1.0.30

.gitignore

@@ -1,5 +1,22 @@
 .nogit/
-node_modules/
+
+# artifacts
 coverage/
 public/
 pages/
+
+# installs
+node_modules/
+
+# caches
+.yarn/
+.cache/
+.rpt2_cache
+
+# builds
+dist/
+dist_web/
+dist_serve/
+dist_ts_web/
+
+# custom

.gitlab-ci.yml

@@ -1,10 +1,10 @@
-# gitzone standard
-image: hosttoday/ht-docker-node:npmci
+# gitzone ci_default
+image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
 
 cache:
   paths:
     - .npmci_cache/
-  key: "$CI_BUILD_STAGE"
+  key: '$CI_BUILD_STAGE'
 
 stages:
   - security
@@ -20,6 +20,7 @@ mirror:
   script:
     - npmci git mirror
   tags:
+    - lossless
     - docker
     - notpriv
 
@@ -31,38 +32,15 @@ snyk:
     - npmci command npm install --ignore-scripts
     - npmci command snyk test
   tags:
+    - lossless
     - docker
     - notpriv
 
 # ====================
 # test stage
 # ====================
-testLEGACY:
-  stage: test
-  script:
-  - npmci npm prepare
-  - npmci node install legacy
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-  allow_failure: true
 
-testLTS:
-  stage: test
-  script:
-  - npmci npm prepare
-  - npmci node install lts
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-    
-testSTABLE:
+testStable:
   stage: test
   script:
     - npmci npm prepare
@@ -71,6 +49,20 @@ testSTABLE:
     - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
+    - lossless
+    - docker
+    - priv
+
+testBuild:
+  stage: test
+  script:
+    - npmci npm prepare
+    - npmci node install stable
+    - npmci npm install
+    - npmci command npm run build
+  coverage: /\d+.?\d+?\%\s*coverage/
+  tags:
+    - lossless
     - docker
     - notpriv
 
@@ -82,6 +74,7 @@ release:
   only:
     - tags
   tags:
+    - lossless
     - docker
     - notpriv
 
@@ -90,20 +83,14 @@ release:
 # ====================
 codequality:
   stage: metadata
-  image: docker:stable
   allow_failure: true
-  services:
-    - docker:stable-dind
   script:
-    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
-    - docker run
-        --env SOURCE_CODE="$PWD"
-        --volume "$PWD":/code
-        --volume /var/run/docker.sock:/var/run/docker.sock
-        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
-  artifacts:
-    paths: [codeclimate.json]
+    - npmci command npm install -g tslint typescript
+    - npmci npm prepare
+    - npmci npm install
+    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
+    - lossless
     - docker
     - priv
 
@@ -114,18 +101,20 @@ trigger:
   only:
     - tags
   tags:
+    - lossless
     - docker
     - notpriv
 
 pages:
-  image: hosttoday/ht-docker-node:npmci
   stage: metadata
   script:
-    - npmci command npm install -g typedoc typescript
+    - npmci node install lts
+    - npmci command npm install -g @gitzone/tsdoc
     - npmci npm prepare
     - npmci npm install
-    - npmci command typedoc --module "commonjs" --target "ES2016" --out public/ ts/
+    - npmci command tsdoc
   tags:
+    - lossless
     - docker
     - notpriv
   only:
@@ -135,13 +124,3 @@ pages:
     paths:
       - public
   allow_failure: true
-
-windowsCompatibility:
-  image: stefanscherer/node-windows:10-build-tools
-  stage: metadata
-  script:
-  - npm install & npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - windows
-  allow_failure: true

npmextra.json

@@ -2,5 +2,15 @@
   "npmci": {
     "npmGlobalTools": [],
     "npmAccessLevel": "public"
+  },
+  "gitzone": {
+    "module": {
+      "githost": "gitlab.com",
+      "gitscope": "mojoio",
+      "gitrepo": "elasticsearch",
+      "shortDescription": "log to elasticsearch in a kibana compatible format",
+      "npmPackagename": "@mojoio/elasticsearch",
+      "license": "MIT"
+    }
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mojoio/elasticsearch",
-  "version": "1.0.19",
+  "version": "1.0.30",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist/index.js",
@@ -13,19 +13,33 @@
     "build": "(tsbuild)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.0.22",
-    "@gitzone/tsrun": "^1.1.13",
-    "@gitzone/tstest": "^1.0.15",
-    "@pushrocks/qenv": "^2.0.2",
-    "@pushrocks/tapbundle": "^3.0.7",
-    "@types/node": "^10.12.2",
-    "tslint": "^5.11.0",
-    "tslint-config-prettier": "^1.15.0"
+    "@gitzone/tsbuild": "^2.1.17",
+    "@gitzone/tsrun": "^1.2.8",
+    "@gitzone/tstest": "^1.0.28",
+    "@pushrocks/qenv": "^4.0.6",
+    "@pushrocks/tapbundle": "^3.0.13",
+    "@types/node": "^12.12.5",
+    "tslint": "^5.20.0",
+    "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "@pushrocks/smartdelay": "^2.0.2",
-    "@pushrocks/smartlog-interfaces": "^2.0.2",
-    "@types/elasticsearch": "^5.0.28",
-    "elasticsearch": "^15.2.0"
-  }
+    "@pushrocks/lik": "^3.0.11",
+    "@pushrocks/smartdelay": "^2.0.3",
+    "@pushrocks/smartlog-interfaces": "^2.0.9",
+    "@pushrocks/smartpromise": "^3.0.6",
+    "@pushrocks/smarttime": "^3.0.12",
+    "@types/elasticsearch": "^5.0.35",
+    "elasticsearch": "^16.5.0"
+  },
+  "files": [
+    "ts/**/*",
+    "ts_web/**/*",
+    "dist/**/*",
+    "dist_web/**/*",
+    "dist_ts_web/**/*",
+    "assets/**/*",
+    "cli.js",
+    "npmextra.json",
+    "readme.md"
+  ]
 }

qenv.yml

@@ -1,4 +1,4 @@
-vars:
+required:
   - ELK_DOMAIN
   - ELK_PORT
   - ELK_USER

readme.md

@@ -1,26 +1,20 @@
-# elasticlog
-
+# @mojoio/elasticsearch
 log to elasticsearch in a kibana compatible format
 
-## Availabililty
-
-[![npm](https://pushrocks.gitlab.io/assets/repo-button-npm.svg)](https://www.npmjs.com/package/elasticlog)
-[![git](https://pushrocks.gitlab.io/assets/repo-button-git.svg)](https://GitLab.com/pushrocks/elasticlog)
-[![git](https://pushrocks.gitlab.io/assets/repo-button-mirror.svg)](https://github.com/pushrocks/elasticlog)
-[![docs](https://pushrocks.gitlab.io/assets/repo-button-docs.svg)](https://pushrocks.gitlab.io/elasticlog/)
+## Availabililty and Links
+* [npmjs.org (npm package)](https://www.npmjs.com/package/@mojoio/elasticsearch)
+* [gitlab.com (source)](https://gitlab.com/mojoio/elasticsearch)
+* [github.com (source mirror)](https://github.com/mojoio/elasticsearch)
+* [docs (typedoc)](https://mojoio.gitlab.io/elasticsearch/)
 
 ## Status for master
-
-[![build status](https://GitLab.com/pushrocks/elasticlog/badges/master/build.svg)](https://GitLab.com/pushrocks/elasticlog/commits/master)
-[![coverage report](https://GitLab.com/pushrocks/elasticlog/badges/master/coverage.svg)](https://GitLab.com/pushrocks/elasticlog/commits/master)
-[![npm downloads per month](https://img.shields.io/npm/dm/elasticlog.svg)](https://www.npmjs.com/package/elasticlog)
-[![Dependency Status](https://david-dm.org/pushrocks/elasticlog.svg)](https://david-dm.org/pushrocks/elasticlog)
-[![bitHound Dependencies](https://www.bithound.io/github/pushrocks/elasticlog/badges/dependencies.svg)](https://www.bithound.io/github/pushrocks/elasticlog/master/dependencies/npm)
-[![bitHound Code](https://www.bithound.io/github/pushrocks/elasticlog/badges/code.svg)](https://www.bithound.io/github/pushrocks/elasticlog)
-[![Known Vulnerabilities](https://snyk.io/test/npm/elasticlog/badge.svg)](https://snyk.io/test/npm/elasticlog)
-[![TypeScript](https://img.shields.io/badge/TypeScript-2.x-blue.svg)](https://nodejs.org/dist/latest-v6.x/docs/api/)
-[![node](https://img.shields.io/badge/node->=%206.x.x-blue.svg)](https://nodejs.org/dist/latest-v6.x/docs/api/)
-[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/)
+[![build status](https://gitlab.com/mojoio/elasticsearch/badges/master/build.svg)](https://gitlab.com/mojoio/elasticsearch/commits/master)
+[![coverage report](https://gitlab.com/mojoio/elasticsearch/badges/master/coverage.svg)](https://gitlab.com/mojoio/elasticsearch/commits/master)
+[![npm downloads per month](https://img.shields.io/npm/dm/@mojoio/elasticsearch.svg)](https://www.npmjs.com/package/@mojoio/elasticsearch)
+[![Known Vulnerabilities](https://snyk.io/test/npm/@mojoio/elasticsearch/badge.svg)](https://snyk.io/test/npm/@mojoio/elasticsearch)
+[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
 
 ## Usage
 
@@ -32,3 +26,14 @@ For further information read the linked docs at the top of this README.
 > | By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
 
 [![repo-footer](https://pushrocks.gitlab.io/assets/repo-footer.svg)](https://push.rocks)
+
+
+## Contribute
+ We are always happy for code contributions. If you are not the code contributing type that is ok. + Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: [Contribute monthly :)](https://lossless.link/contribute)
+
+For further information read the linked docs at the top of this readme.
+
+> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
+| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+
+[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)

test/test.ts

@@ -8,11 +8,11 @@ let testElasticLog: elasticsearch.ElasticSearch<any>;
 
 tap.test('first test', async () => {
   testElasticLog = new elasticsearch.ElasticSearch({
-    domain: process.env.ELK_DOMAIN,
-    port: parseInt(process.env.ELK_PORT, 10),
-    ssl: true,
-    user: process.env.ELK_USER,
-    pass: process.env.ELK_PASS
+    indexPrefix: 'smartlog',
+    indexRetention: 7,
+    domain: testQenv.getEnvVarOnDemand('ELK_DOMAIN'),
+    port: parseInt(testQenv.getEnvVarOnDemand('ELK_PORT'), 10),
+    ssl: true
   });
   expect(testElasticLog).to.be.instanceOf(elasticsearch.ElasticSearch);
 });
@@ -30,7 +30,7 @@ tap.test('should send a message to Elasticsearch', async () => {
       runtime: 'node',
       zone: 'ship.zone'
     },
-    message: 'hi, this is a testMessage'
+    message: 'GET https://myroute.to.a.cool.destination/sorare?hello=there'
   });
 });
 

ts/elasticsearch.classes.elasticindex.ts

@@ -0,0 +1,103 @@
+import * as plugins from './elasticsearch.plugins';
+import { ElasticSearch } from './elasticsearch.classes.elasticsearch';
+import { ILogPackage } from '@pushrocks/smartlog-interfaces';
+
+import { Stringmap } from '@pushrocks/lik';
+
+export class ElasticIndex {
+  private stringmap = new Stringmap();
+  private elasticSearchRef: ElasticSearch<any>;
+
+  constructor(elasticSearchInstanceArg: ElasticSearch<ILogPackage>) {
+    this.elasticSearchRef = elasticSearchInstanceArg;
+  }
+
+  public async ensureIndex(indexArg: string) {
+    const done = plugins.smartpromise.defer();
+    if (this.stringmap.checkString(indexArg)) {
+      done.resolve();
+      return;
+    }
+    this.elasticSearchRef.client.cat.indices(
+      {
+        format: 'json',
+        bytes: 'm'
+      },
+      async (err, responseArg: any[]) => {
+        if (err) {
+          console.log(err);
+          return;
+        }
+
+        // lets delete indexes that violate the retention
+        if (Array.isArray(responseArg)) {
+          const filteredIndices = responseArg.filter(indexObjectArg => {
+            return indexObjectArg.index.startsWith('smartlog');
+          });
+          const filteredIndexNames = filteredIndices.map(indexObjectArg => {
+            return indexObjectArg.index;
+          });
+          this.deleteOldIndices(filteredIndexNames);
+        }
+
+        let index = null;
+
+        if (Array.isArray(responseArg)) {
+          index = responseArg.find(indexObject => {
+            return indexObject.index === indexArg;
+          });
+        }
+
+        if (!index) {
+          const done2 = plugins.smartpromise.defer();
+          this.elasticSearchRef.client.indices.create(
+            {
+              waitForActiveShards: '1',
+              index: indexArg
+            },
+            (error, response) => {
+              // console.lof(response)
+              done2.resolve();
+            }
+          );
+          await done2.promise;
+        }
+        this.stringmap.addString(indexArg);
+        done.resolve();
+      }
+    );
+    await done.promise;
+  }
+
+  public createNewIndex(indexNameArg: string) {}
+
+  public async deleteOldIndices(indicesArray: string[]) {
+    const todayAsUnix: number = Date.now();
+    const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
+      this.elasticSearchRef.indexRetention
+    );
+    for (const indexName of indicesArray) {
+      const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName);
+      const dateAsUnix: number = new Date(
+        `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
+      ).getTime();
+      if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
+        console.log(`found old index ${indexName}`);
+        const done2 = plugins.smartpromise.defer();
+        this.elasticSearchRef.client.indices.delete(
+          {
+            index: indexName
+          },
+          (err2, response2) => {
+            if (err2) {
+              console.log(err2);
+            }
+            console.log(`deleted ${indexName}`);
+            done2.resolve();
+          }
+        );
+        await done2.promise;
+      }
+    }
+  }
+}

ts/elasticsearch.classes.elasticscheduler.ts

@@ -0,0 +1,44 @@
+import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch';
+
+export class ElasticScheduler {
+  elasticSearchRef: ElasticSearch<any>;
+  docsScheduled = false;
+  docsStorage: any[] = [];
+
+  constructor(elasticLogRefArg: ElasticSearch<any>) {
+    this.elasticSearchRef = elasticLogRefArg;
+  }
+
+  public addFailedDoc(objectArg: any | IStandardLogParams) {
+    this.docsStorage.push(objectArg);
+    this.setRetry();
+  }
+  public scheduleDoc(logObject: any) {
+    this.docsStorage.push(logObject);
+  }
+
+  public setRetry() {
+    setTimeout(() => {
+      const oldStorage = this.docsStorage;
+      this.docsStorage = [];
+      for (let logObject of oldStorage) {
+        this.elasticSearchRef.log(logObject, true);
+      }
+      if (this.docsStorage.length === 0) {
+        console.log('ElasticLog retry success!!!');
+        this.docsScheduled = false;
+      } else {
+        console.log('ElasticLog retry failed');
+        this.setRetry();
+      }
+    }, 5000);
+  }
+
+  public deferSend() {
+    if (!this.docsScheduled) {
+      console.log('Retry ElasticLog in 5 seconds!');
+      this.docsScheduled = true;
+      this.setRetry();
+    }
+  }
+}

ts/elasticsearch.classes.elasticsearch.ts

@@ -3,14 +3,17 @@ import { Client as ElasticClient } from 'elasticsearch';
 import { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
 
 // other classes
-import { LogScheduler } from './elasticsearch.classes.logscheduler';
+import { ElasticScheduler } from './elasticsearch.classes.elasticscheduler';
+import { ElasticIndex } from './elasticsearch.classes.elasticindex';
 
 export interface IStandardLogParams {
   message: string;
   severity: string;
 }
 
-export interface IElasticLogConstructorOptions {
+export interface IElasticSearchConstructorOptions {
+  indexPrefix: string;
+  indexRetention: number;
   port: number;
   domain: string;
   ssl: boolean;
@@ -19,25 +22,31 @@ export interface IElasticLogConstructorOptions {
 }
 
 export class ElasticSearch<T> {
-  client: ElasticClient;
-  logScheduler = new LogScheduler(this);
+  public client: ElasticClient;
+  public elasticScheduler = new ElasticScheduler(this);
+  public elasticIndex: ElasticIndex = new ElasticIndex(this);
+
+  public indexPrefix: string;
+  public indexRetention: number;
 
   /**
    * sets up an instance of Elastic log
    * @param optionsArg
    */
-  constructor(optionsArg: IElasticLogConstructorOptions) {
+  constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
-      host: this.computeHostString(optionsArg),
-      log: 'trace'
+      host: this.computeHostString(optionsArg)
+      // log: 'trace'
     });
+    this.indexPrefix = optionsArg.indexPrefix;
+    this.indexRetention = optionsArg.indexRetention;
   }
 
   /**
    * computes the host string from the constructor options
    * @param optionsArg
    */
-  private computeHostString(optionsArg: IElasticLogConstructorOptions): string {
+  private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
     let hostString = `${optionsArg.domain}:${optionsArg.port}`;
     if (optionsArg.user && optionsArg.pass) {
       hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
@@ -47,20 +56,27 @@ export class ElasticSearch<T> {
     } else {
       hostString = `http://${hostString}`;
     }
+    console.log(hostString);
     return hostString;
   }
 
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    if (this.logScheduler.logsScheduled && !scheduleOverwrite) {
-      this.logScheduler.scheduleLog(logPackageArg);
+    const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${(
+      '0' +
+      (now.getMonth() + 1)
+    ).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
+
+    if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
+      this.elasticScheduler.scheduleDoc(logPackageArg);
       return;
     }
+
+    await this.elasticIndex.ensureIndex(indexToUse);
+
     this.client.index(
       {
-        index: `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
-          '0' + now.getDate()
-        ).slice(-2)}`,
+        index: indexToUse,
         type: 'log',
         body: {
           '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
@@ -71,9 +87,9 @@ export class ElasticSearch<T> {
         if (error) {
           console.log('ElasticLog encountered an error:');
           console.log(error);
-          this.logScheduler.addFailedLog(logPackageArg);
+          this.elasticScheduler.addFailedDoc(logPackageArg);
         } else {
-          console.log(`ElasticLog: ${logPackageArg.message}`);
+          // console.log(`ElasticLog: ${logPackageArg.message}`);
         }
       }
     );
@@ -84,6 +100,6 @@ export class ElasticSearch<T> {
       handleLog: (smartlogPackageArg: ILogPackage) => {
         this.log(smartlogPackageArg);
       }
-    }
+    };
   }
 }

ts/elasticsearch.classes.logscheduler.ts

@@ -1,44 +0,0 @@
-import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch';
-
-export class LogScheduler {
-  elasticLogRef: ElasticSearch<any>;
-  logsScheduled = false;
-  logStorage: any[] = [];
-
-  constructor(elasticLogRefArg: ElasticSearch<any>) {
-    this.elasticLogRef = elasticLogRefArg;
-  }
-
-  addFailedLog(objectArg: any | IStandardLogParams) {
-    this.logStorage.push(objectArg);
-    this.setRetry();
-  }
-  scheduleLog(logObject: any) {
-    this.logStorage.push(logObject);
-  }
-
-  setRetry() {
-    setTimeout(() => {
-      const oldStorage = this.logStorage;
-      this.logStorage = [];
-      for (let logObject of oldStorage) {
-        this.elasticLogRef.log(logObject, true);
-      }
-      if (this.logStorage.length === 0) {
-        console.log('ElasticLog retry success!!!');
-        this.logsScheduled = false;
-      } else {
-        console.log('ElasticLog retry failed');
-        this.setRetry();
-      }
-    }, 5000);
-  }
-
-  deferSend() {
-    if (!this.logsScheduled) {
-      console.log('Retry ElasticLog in 5 seconds!');
-      this.logsScheduled = true;
-      this.setRetry();
-    }
-  }
-}

ts/elasticsearch.plugins.ts

@@ -1,4 +1,8 @@
 import * as elasticsearch from 'elasticsearch';
+import * as lik from '@pushrocks/lik';
 import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
-export { elasticsearch, smartdelay, smartlogInterfaces };
+import * as smartpromise from '@pushrocks/smartpromise';
+import * as smarttime from '@pushrocks/smarttime';
+
+export { elasticsearch, lik, smartdelay, smartlogInterfaces, smartpromise, smarttime };