1.0.46

.gitlab-ci.yml

@@ -12,29 +12,38 @@ stages:
   - release
   - metadata
 
+before_script:
+  - pnpm install -g pnpm
+  - pnpm install -g @shipzone/npmci
+  - npmci npm prepare
+
 # ====================
 # security stage
 # ====================
-mirror:
+# ====================
+# security stage
+# ====================
+auditProductionDependencies:
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci git mirror
+     - npmci command npm config set registry https://registry.npmjs.org
+     - npmci command pnpm audit --audit-level=high --prod
   tags:
     - lossless
     - docker
-    - notpriv
+  allow_failure: true
 
-snyk:
+auditDevDependencies:
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
-    - npmci npm prepare
-    - npmci command npm install -g snyk
-    - npmci command npm install --ignore-scripts
-    - npmci command snyk test
+    - npmci command npm config set registry https://registry.npmjs.org
+    - npmci command pnpm audit --audit-level=high --dev
   tags:
     - lossless
     - docker
-    - notpriv
+  allow_failure: true
 
 # ====================
 # test stage
@@ -43,33 +52,24 @@ snyk:
 testStable:
   stage: test
   script:
-    - npmci npm prepare
-    - npmci node install stable
     - npmci npm install
     - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - priv
 
 testBuild:
   stage: test
   script:
-    - npmci npm prepare
-    - npmci node install stable
     - npmci npm install
-    - npmci command npm run build
+    - npmci npm build
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - notpriv
 
 release:
   stage: release
   script:
-    - npmci node install stable
     - npmci npm publish
   only:
     - tags
@@ -84,11 +84,12 @@ release:
 codequality:
   stage: metadata
   allow_failure: true
+  only:
+    - tags
   script:
-    - npmci command npm install -g tslint typescript
+    - npmci command npm install -g typescript
     - npmci npm prepare
     - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
     - lossless
     - docker
@@ -108,11 +109,9 @@ trigger:
 pages:
   stage: metadata
   script:
-    - npmci node install lts
-    - npmci command npm install -g @gitzone/tsdoc
-    - npmci npm prepare
+    - npmci node install stable
     - npmci npm install
-    - npmci command tsdoc
+    - npmci command npm run buildDocs
   tags:
     - lossless
     - docker

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@mojoio/elasticsearch",
-  "version": "1.0.32",
+  "name": "@apiclient.xyz/elasticsearch",
+  "version": "1.0.46",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist_ts/index.js",
@@ -10,7 +10,7 @@
   "scripts": {
     "test": "(tstest test/)",
     "format": "(gitzone format)",
-    "build": "(tsbuild)",
+    "build": "(tsbuild --allowimplicitany)",
     "buildDocs": "tsdoc"
   },
   "devDependencies": {
@@ -22,13 +22,12 @@
     "@types/node": "^20.3.3"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "7.17.11-patch.1",
     "@pushrocks/lik": "^6.0.2",
     "@pushrocks/smartdelay": "^3.0.1",
     "@pushrocks/smartlog-interfaces": "^3.0.0",
     "@pushrocks/smartpromise": "^4.0.2",
-    "@pushrocks/smarttime": "^4.0.1",
-    "@types/elasticsearch": "^5.0.40",
-    "elasticsearch": "^16.7.3"
+    "@pushrocks/smarttime": "^4.0.1"
   },
   "files": [
     "ts/**/*",

pnpm-lock.yaml

@@ -5,6 +5,9 @@ settings:
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@elastic/elasticsearch':
+    specifier: 7.17.11-patch.1
+    version: 7.17.11-patch.1
   '@pushrocks/lik':
     specifier: ^6.0.2
     version: 6.0.2
@@ -20,12 +23,6 @@ dependencies:
   '@pushrocks/smarttime':
     specifier: ^4.0.1
     version: 4.0.1
-  '@types/elasticsearch':
-    specifier: ^5.0.40
-    version: 5.0.40
-  elasticsearch:
-    specifier: ^16.7.3
-    version: 16.7.3
 
 devDependencies:
   '@gitzone/tsbuild':
@@ -124,6 +121,18 @@ packages:
       '@jridgewell/trace-mapping': 0.3.9
     dev: true
 
+  /@elastic/elasticsearch@7.17.11-patch.1:
+    resolution: {integrity: sha512-t0twlzgC+fCgvw1s8jI2GFO3rpxV/V2gHTr3iHeGBDNeQVsRZ+6Psy8P7SEXW/uTvxuDfmUF6UShHhK1uHXyow==}
+    engines: {node: '>=12'}
+    dependencies:
+      debug: 4.3.4
+      hpagent: 0.1.2
+      ms: 2.1.3
+      secure-json-parse: 2.7.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /@esbuild/android-arm64@0.17.19:
     resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==}
     engines: {node: '>=12'}
@@ -1232,10 +1241,6 @@ packages:
     resolution: {integrity: sha512-tpu0hp+AOIzwdAHyZPzLE5pCf9uT0pb+xZ76T4S7MrY2YTVq918Q7Q2VQ3KCVQqYxM7nxuCK/SL3X97jBEIeKQ==}
     dev: true
 
-  /@types/elasticsearch@5.0.40:
-    resolution: {integrity: sha512-lhnbkC0XorAD7Dt7X+94cXUSHEdDNnEVk/DgFLHgIZQNhixV631Lj4+KpXunTT5rCHyj9RqK3TfO7QrOiwEeUQ==}
-    dev: false
-
   /@types/express-serve-static-core@4.17.35:
     resolution: {integrity: sha512-wALWQwrgiB2AWTT91CB62b6Yt0sNHpznUXeZEcnPU3DRdlDIz74x8Qg1UUYKSVFi+va5vKOLYRBI1bRKiLLKIg==}
     dependencies:
@@ -1568,6 +1573,11 @@ packages:
       - utf-8-validate
     dev: true
 
+  /@xmldom/xmldom@0.8.8:
+    resolution: {integrity: sha512-0LNz4EY8B/8xXY86wMrQ4tz6zEHZv9ehFMJPm8u2gq5lQ71cfRKdaKyxfJAx5aUoyzx0qzgURblTisPGgz3d+Q==}
+    engines: {node: '>=10.0.0'}
+    dev: true
+
   /accepts@1.3.8:
     resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
     engines: {node: '>= 0.6'}
@@ -1596,13 +1606,6 @@ packages:
       - supports-color
     dev: true
 
-  /agentkeepalive@3.5.2:
-    resolution: {integrity: sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==}
-    engines: {node: '>= 4.0.0'}
-    dependencies:
-      humanize-ms: 1.2.1
-    dev: false
-
   /agentkeepalive@4.3.0:
     resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==}
     engines: {node: '>= 8.0.0'}
@@ -1634,11 +1637,6 @@ packages:
       type-fest: 0.21.3
     dev: true
 
-  /ansi-regex@2.1.1:
-    resolution: {integrity: sha1-w7M6te42DYbg5ijwRorn7yfWVN8=}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   /ansi-regex@5.0.1:
     resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
     engines: {node: '>=8'}
@@ -1649,11 +1647,6 @@ packages:
     engines: {node: '>=12'}
     dev: true
 
-  /ansi-styles@2.2.1:
-    resolution: {integrity: sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   /ansi-styles@3.2.1:
     resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
     engines: {node: '>=4'}
@@ -1886,17 +1879,6 @@ packages:
       axe-core: 4.7.2
     dev: true
 
-  /chalk@1.1.3:
-    resolution: {integrity: sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-styles: 2.2.1
-      escape-string-regexp: 1.0.5
-      has-ansi: 2.0.0
-      strip-ansi: 3.0.1
-      supports-color: 2.0.0
-    dev: false
-
   /chalk@2.4.2:
     resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
     engines: {node: '>=4'}
@@ -2131,7 +2113,6 @@ packages:
         optional: true
     dependencies:
       ms: 2.1.2
-    dev: true
 
   /decompress-response@6.0.0:
     resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
@@ -2234,16 +2215,6 @@ packages:
     resolution: {integrity: sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=}
     dev: true
 
-  /elasticsearch@16.7.3:
-    resolution: {integrity: sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==}
-    engines: {node: '>=0.8'}
-    deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.
-    dependencies:
-      agentkeepalive: 3.5.2
-      chalk: 1.1.3
-      lodash: 4.17.21
-    dev: false
-
   /emoji-regex@8.0.0:
     resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
     dev: true
@@ -2371,6 +2342,7 @@ packages:
   /escape-string-regexp@1.0.5:
     resolution: {integrity: sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=}
     engines: {node: '>=0.8.0'}
+    dev: true
 
   /escape-string-regexp@4.0.0:
     resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
@@ -2704,13 +2676,6 @@ packages:
     resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
     dev: true
 
-  /has-ansi@2.0.0:
-    resolution: {integrity: sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-regex: 2.1.1
-    dev: false
-
   /has-flag@3.0.0:
     resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=}
     engines: {node: '>=4'}
@@ -2752,6 +2717,10 @@ packages:
     hasBin: true
     dev: true
 
+  /hpagent@0.1.2:
+    resolution: {integrity: sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==}
+    dev: false
+
   /html-escaper@2.0.2:
     resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
     dev: true
@@ -2836,6 +2805,7 @@ packages:
     resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=}
     dependencies:
       ms: 2.1.3
+    dev: true
 
   /iconv-lite@0.4.24:
     resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
@@ -3253,6 +3223,7 @@ packages:
 
   /lodash@4.17.21:
     resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
+    dev: true
 
   /log-update@4.0.0:
     resolution: {integrity: sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==}
@@ -3421,7 +3392,6 @@ packages:
 
   /ms@2.1.2:
     resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-    dev: true
 
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@@ -3637,6 +3607,8 @@ packages:
     resolution: {integrity: sha512-mXF9AIgnvq1DP/ZM2R28tAfxP2wKZHYa2DjV0R1KCwcqSzm5Iqh1XQq9rdfAt6dp2DuPP0VHZIaCALc2v1cL5A==}
     engines: {node: '>=14.18.0', npm: '>=6.14.15'}
     hasBin: true
+    dependencies:
+      '@xmldom/xmldom': 0.8.8
     dev: true
     bundledDependencies:
       - '@xmldom/xmldom'
@@ -3934,6 +3906,10 @@ packages:
     resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==}
     dev: true
 
+  /secure-json-parse@2.7.0:
+    resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==}
+    dev: false
+
   /semver@6.3.0:
     resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==}
     hasBin: true
@@ -4143,13 +4119,6 @@ packages:
       safe-buffer: 5.2.1
     dev: true
 
-  /strip-ansi@3.0.1:
-    resolution: {integrity: sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      ansi-regex: 2.1.1
-    dev: false
-
   /strip-ansi@6.0.1:
     resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
     engines: {node: '>=8'}
@@ -4175,11 +4144,6 @@ packages:
     resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}
     dev: true
 
-  /supports-color@2.0.0:
-    resolution: {integrity: sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=}
-    engines: {node: '>=0.8.0'}
-    dev: false
-
   /supports-color@5.5.0:
     resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
     engines: {node: '>=4'}

qenv.yml

@@ -1,5 +1,2 @@
 required:
-  - ELK_DOMAIN
-  - ELK_PORT
-  - ELK_USER
-  - ELK_PASS
+  

readme.md

@@ -37,7 +37,6 @@ For further information read the linked docs at the top of this README.
 
 We are always happy for code contributions. If you are not the code contributing type that is ok. + Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: [Contribute monthly :)](https://lossless.link/contribute)
 
-
 ## Contribution
 
 We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)

test/test.nonci.ts

@@ -2,23 +2,24 @@ import { expect, tap } from '@pushrocks/tapbundle';
 import { Qenv } from '@pushrocks/qenv';
 import * as elasticsearch from '../ts/index.js';
 
-const testQenv = new Qenv('./', './.nogit/');
-
-let testElasticLog: elasticsearch.ElasticSearch<any>;
+let testElasticLog: elasticsearch.ElsSmartlogDestination<any>;
+let testElasticDoc: elasticsearch.ElasticDoc;
 
 tap.test('first test', async () => {
-  testElasticLog = new elasticsearch.ElasticSearch({
-    indexPrefix: 'smartlog',
+  testElasticLog = new elasticsearch.ElsSmartlogDestination({
+    indexPrefix: 'testprefix',
     indexRetention: 7,
-    domain: testQenv.getEnvVarOnDemand('ELK_DOMAIN'),
-    port: parseInt(testQenv.getEnvVarOnDemand('ELK_PORT'), 10),
-    ssl: true,
+    node: 'http://localhost:9200',
+    auth: {
+      username: 'elastic',
+      password: 'YourPassword'
+    }
   });
-  expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch);
+  expect(testElasticLog).toBeInstanceOf(elasticsearch.ElsSmartlogDestination);
 });
 
 tap.test('should send a message to Elasticsearch', async () => {
-  testElasticLog.log({
+  await testElasticLog.log({
     timestamp: Date.now(),
     type: 'increment',
     level: 'info',
@@ -35,4 +36,27 @@ tap.test('should send a message to Elasticsearch', async () => {
   });
 });
 
+tap.test('should create an ElasticDoc instance', async () => {
+  testElasticDoc = new elasticsearch.ElasticDoc({
+    index: 'testindex',
+    node: 'http://localhost:9200',
+    auth: {
+      username: 'elastic',
+      password: 'YourPassword'
+    }
+  });
+  expect(testElasticDoc).toBeInstanceOf(elasticsearch.ElasticDoc);
+});
+
+tap.test('should add and update documents in a piping session', async () => {
+  await testElasticDoc.startPipingSession();
+  await testElasticDoc.pipeDocument('1', { name: 'doc1' });
+  await testElasticDoc.pipeDocument('2', { name: 'doc2' });
+  await testElasticDoc.pipeDocument('1', { name: 'updated doc1' });
+});
+
+tap.test('should delete documents not part of the piping session', async () => {
+  await testElasticDoc.endPipingSession();
+});
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -2,7 +2,7 @@
  * autocreated commitinfo by @pushrocks/commitinfo
  */
 export const commitinfo = {
-  name: '@mojoio/elasticsearch',
-  version: '1.0.32',
+  name: '@apiclient.xyz/elasticsearch',
+  version: '1.0.46',
   description: 'log to elasticsearch in a kibana compatible format'
 }

ts/elasticsearch.classes.elasticindex.ts

@@ -1,102 +1,102 @@
-import * as plugins from './elasticsearch.plugins';
-import { ElasticSearch } from './elasticsearch.classes.elasticsearch';
-import { ILogPackage } from '@pushrocks/smartlog-interfaces';
-
+import * as plugins from './elasticsearch.plugins.js';
+import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js';
+import { type ILogPackage } from '@pushrocks/smartlog-interfaces';
 import { Stringmap } from '@pushrocks/lik';
 
 export class ElasticIndex {
   private stringmap = new Stringmap();
-  private elasticSearchRef: ElasticSearch<any>;
+  private elasticSearchRef: ElsSmartlogDestination<any>;
 
-  constructor(elasticSearchInstanceArg: ElasticSearch<ILogPackage>) {
+  constructor(elasticSearchInstanceArg: ElsSmartlogDestination<ILogPackage>) {
     this.elasticSearchRef = elasticSearchInstanceArg;
   }
 
-  public async ensureIndex(indexArg: string) {
-    const done = plugins.smartpromise.defer();
-    if (this.stringmap.checkString(indexArg)) {
-      done.resolve();
-      return;
+  public async ensureIndex(prefixArg: string, indexNameArg: string) {
+    if (this.stringmap.checkString(indexNameArg)) {
+      return indexNameArg;
     }
-    this.elasticSearchRef.client.cat.indices(
-      {
-        format: 'json',
-        bytes: 'm',
-      },
-      async (err, responseArg: any[]) => {
-        if (err) {
-          console.log(err);
-          return;
-        }
 
-        // lets delete indexes that violate the retention
-        if (Array.isArray(responseArg)) {
-          const filteredIndices = responseArg.filter((indexObjectArg) => {
-            return indexObjectArg.index.startsWith('smartlog');
-          });
-          const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
-            return indexObjectArg.index;
-          });
-          this.deleteOldIndices(filteredIndexNames);
-        }
+    const responseArg = await this.elasticSearchRef.client.cat.indices({
+      format: 'json',
+      bytes: 'm',
+    }).catch(err => {
+      console.log(err);
+    });
 
-        let index = null;
+    if (!responseArg) {
+      throw new Error('Could not get valid response from elastic search');
+    }
 
-        if (Array.isArray(responseArg)) {
-          index = responseArg.find((indexObject) => {
-            return indexObject.index === indexArg;
-          });
-        }
+    if (Array.isArray(responseArg.body)) {
+      const filteredIndices = responseArg.body.filter((indexObjectArg) => {
+        return indexObjectArg.index.startsWith(prefixArg);
+      });
+      const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
+        return indexObjectArg.index;
+      });
+      await this.deleteOldIndices(prefixArg, filteredIndexNames);
+    }
 
-        if (!index) {
-          const done2 = plugins.smartpromise.defer();
-          this.elasticSearchRef.client.indices.create(
-            {
-              waitForActiveShards: '1',
-              index: indexArg,
-            },
-            (error, response) => {
-              // console.lof(response)
-              done2.resolve();
-            }
-          );
-          await done2.promise;
-        }
-        this.stringmap.addString(indexArg);
-        done.resolve();
-      }
-    );
-    await done.promise;
+    let index = null;
+
+    if (Array.isArray(responseArg.body)) {
+      index = responseArg.body.find((indexItemArg) => {
+        return indexItemArg.index === indexNameArg;
+      });
+    }
+
+    if (!index) {
+      await this.createNewIndex(indexNameArg);
+    }
+
+    this.stringmap.addString(indexNameArg);
+    return index;
   }
 
-  public createNewIndex(indexNameArg: string) {}
+  public async createNewIndex(indexNameArg: string) {
+    const response = await this.elasticSearchRef.client.indices.create({
+      wait_for_active_shards: '1',
+      index: indexNameArg,
+      body: {
+        mappings: {
+          properties: {
+            '@timestamp': {
+              type: 'date',
+            },
+            logPackageArg: {
+              properties: {
+                payload: {
+                  type: 'object',
+                  dynamic: true
+                }
+              }
+            },
+          },
+        },
+      },
+    });
+  }
 
-  public async deleteOldIndices(indicesArray: string[]) {
+  public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
     const todayAsUnix: number = Date.now();
     const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
       this.elasticSearchRef.indexRetention
     );
     for (const indexName of indicesArray) {
-      const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName);
+      if (!indexName.startsWith(prefixArg)) continue;
+      const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`) 
+      const regexResult = indexRegex.exec(indexName);
       const dateAsUnix: number = new Date(
         `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
       ).getTime();
       if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
         console.log(`found old index ${indexName}`);
-        const done2 = plugins.smartpromise.defer();
-        this.elasticSearchRef.client.indices.delete(
+        const response = await this.elasticSearchRef.client.indices.delete(
           {
             index: indexName,
-          },
-          (err2, response2) => {
-            if (err2) {
-              console.log(err2);
-            }
-            console.log(`deleted ${indexName}`);
-            done2.resolve();
-          }
-        );
-        await done2.promise;
+          }).catch(err => {
+            console.log(err);
+          });
       }
     }
   }

ts/elasticsearch.classes.elasticscheduler.ts

@@ -1,29 +1,48 @@
-import { ElasticSearch, type IStandardLogParams } from './elasticsearch.classes.elasticsearch.js';
+import { ElsSmartlogDestination, type IStandardLogParams } from './els.classes.smartlogdestination.js';
 
 export class ElasticScheduler {
-  elasticSearchRef: ElasticSearch<any>;
+  elasticSearchRef: ElsSmartlogDestination<any>;
   docsScheduled = false;
   docsStorage: any[] = [];
 
-  constructor(elasticLogRefArg: ElasticSearch<any>) {
+  // maximum size of the buffer
+  maxBufferSize = 500;
+
+  constructor(elasticLogRefArg: ElsSmartlogDestination<any>) {
     this.elasticSearchRef = elasticLogRefArg;
   }
 
   public addFailedDoc(objectArg: any | IStandardLogParams) {
-    this.docsStorage.push(objectArg);
+    this.addToStorage(objectArg);
     this.setRetry();
   }
+  
   public scheduleDoc(logObject: any) {
+    this.addToStorage(logObject);
+  }
+
+  private addToStorage(logObject: any) {
     this.docsStorage.push(logObject);
+    
+    // if buffer is full, send logs immediately
+    if (this.docsStorage.length >= this.maxBufferSize) {
+      this.flushLogsToElasticSearch();
+    }
+  }
+
+  private flushLogsToElasticSearch() {
+    const oldStorage = this.docsStorage;
+    this.docsStorage = [];
+    
+    for (let logObject of oldStorage) {
+      this.elasticSearchRef.log(logObject, true);
+    }
   }
 
   public setRetry() {
     setTimeout(() => {
-      const oldStorage = this.docsStorage;
-      this.docsStorage = [];
-      for (let logObject of oldStorage) {
-        this.elasticSearchRef.log(logObject, true);
-      }
+      this.flushLogsToElasticSearch();
+
       if (this.docsStorage.length === 0) {
         console.log('ElasticLog retry success!!!');
         this.docsScheduled = false;

ts/elasticsearch.plugins.ts

@@ -1,4 +1,4 @@
-import * as elasticsearch from 'elasticsearch';
+import * as elasticsearch from '@elastic/elasticsearch';
 import * as lik from '@pushrocks/lik';
 import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';

ts/els.classes.elasticdoc.ts

@@ -0,0 +1,77 @@
+import { Client as ElasticClient } from '@elastic/elasticsearch';
+
+export interface IElasticDocConstructorOptions {
+  index: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
+}
+
+export class ElasticDoc {
+  public client: ElasticClient;
+  public index: string;
+  private sessionDocs: Set<string> = new Set();
+  
+  private BATCH_SIZE = 1000;
+
+  constructor(options: IElasticDocConstructorOptions) {
+    this.client = new ElasticClient({
+      node: options.node,
+      ...(options.auth && { auth: options.auth }),
+    });
+    this.index = options.index;
+  }
+
+  async startPipingSession() {
+    // Clear the session docs set
+    this.sessionDocs.clear();
+  }
+
+  async pipeDocument(docId: string, doc: any) {
+    await this.client.index({
+      index: this.index,
+      id: docId,
+      body: doc,
+    });
+    this.sessionDocs.add(docId);
+  }
+
+  async endPipingSession() {
+    const allDocIds: string[] = [];
+    const responseQueue = [];
+    let response = await this.client.search({ index: this.index, scroll: '1m', size: this.BATCH_SIZE });
+    while (true) {
+      response.body.hits.hits.forEach((hit: any) => allDocIds.push(hit._id));
+      if (!response.body.hits.hits.length) {
+        break;
+      }
+      response = await this.client.scroll({ scroll_id: response.body._scroll_id, scroll: '1m' });
+    }
+
+    // Batch delete docs
+    for (const docId of allDocIds) {
+      if (!this.sessionDocs.has(docId)) {
+        responseQueue.push({
+          delete: {
+            _index: this.index,
+            _id: docId,
+          },
+        });
+
+        if (responseQueue.length >= this.BATCH_SIZE) {
+          await this.client.bulk({ refresh: true, body: responseQueue });
+          responseQueue.length = 0;
+        }
+      }
+    }
+
+    if (responseQueue.length > 0) {
+      await this.client.bulk({ refresh: true, body: responseQueue });
+    }
+
+    // Clear the session docs set
+    this.sessionDocs.clear();
+  }
+}

ts/els.classes.smartlogdestination.ts

@@ -1,5 +1,5 @@
 // interfaces
-import { Client as ElasticClient } from 'elasticsearch';
+import { Client as ElasticClient } from '@elastic/elasticsearch';
 import type { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
 
 // other classes
@@ -14,14 +14,14 @@ export interface IStandardLogParams {
 export interface IElasticSearchConstructorOptions {
   indexPrefix: string;
   indexRetention: number;
-  port: number;
-  domain: string;
-  ssl: boolean;
-  user?: string;
-  pass?: string;
+  node: string;
+  auth?: {
+    username: string;
+    password: string;
+  };
 }
 
-export class ElasticSearch<T> {
+export class ElsSmartlogDestination<T> {
   public client: ElasticClient;
   public elasticScheduler = new ElasticScheduler(this);
   public elasticIndex: ElasticIndex = new ElasticIndex(this);
@@ -35,49 +35,50 @@ export class ElasticSearch<T> {
    */
   constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
-      host: this.computeHostString(optionsArg),
-      // log: 'trace'
+      node: optionsArg.node,
+      ...(optionsArg.auth && { auth: optionsArg.auth }),
     });
-    this.indexPrefix = optionsArg.indexPrefix;
+    this.indexPrefix = `${optionsArg.indexPrefix}`;
     this.indexRetention = optionsArg.indexRetention;
   }
 
-  /**
-   * computes the host string from the constructor options
-   * @param optionsArg
-   */
-  private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
-    let hostString = `${optionsArg.domain}:${optionsArg.port}`;
-    if (optionsArg.user && optionsArg.pass) {
-      hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
-    }
-    if (optionsArg.ssl) {
-      hostString = `https://${hostString}`;
-    } else {
-      hostString = `http://${hostString}`;
-    }
-    console.log(hostString);
-    return hostString;
-  }
-
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${(
-      '0' +
-      (now.getMonth() + 1)
-    ).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
+    const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`;
 
     if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
       this.elasticScheduler.scheduleDoc(logPackageArg);
       return;
     }
 
-    await this.elasticIndex.ensureIndex(indexToUse);
+    // Make sure the index is created with a mapping for dynamic JSON
+    const indexExists = await this.client.indices.exists({ index: indexToUse });
+    if (!indexExists.body) {
+      await this.client.indices.create({
+        index: indexToUse,
+        body: {
+          mappings: {
+            properties: {
+              '@timestamp': {
+                type: 'date',
+              },
+              logPackageArg: {
+                properties: {
+                  payload: {
+                    type: 'object',
+                    dynamic: true
+                  }
+                }
+              },
+            },
+          },
+        },
+      });
+    }
 
     this.client.index(
       {
         index: indexToUse,
-        type: 'log',
         body: {
           '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
           ...logPackageArg,

ts/index.ts

@@ -1 +1,2 @@
-export * from './elasticsearch.classes.elasticsearch.js';
+export * from './els.classes.smartlogdestination.js';
+export * from './els.classes.elasticdoc.js';