1.0.27

.gitlab-ci.yml

@@ -37,18 +37,6 @@ snyk:
 # ====================
 # test stage
 # ====================
-testLEGACY:
-  stage: test
-  script:
-  - npmci npm prepare
-  - npmci node install legacy
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-  allow_failure: true
 
 testLTS:
   stage: test
@@ -135,13 +123,3 @@ pages:
     paths:
     - public
   allow_failure: true
-
-windowsCompatibility:
-  image: stefanscherer/node-windows:10-build-tools
-  stage: metadata
-  script:
-  - npm install & npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - windows
-  allow_failure: true

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mojoio/elasticsearch",
-  "version": "1.0.20",
+  "version": "1.0.27",
   "private": false,
   "description": "log to elasticsearch in a kibana compatible format",
   "main": "dist/index.js",
@@ -13,21 +13,22 @@
     "build": "(tsbuild)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.0.22",
+    "@gitzone/tsbuild": "^2.1.17",
-    "@gitzone/tsrun": "^1.1.13",
+    "@gitzone/tsrun": "^1.2.8",
-    "@gitzone/tstest": "^1.0.15",
+    "@gitzone/tstest": "^1.0.28",
-    "@pushrocks/qenv": "^2.0.2",
+    "@pushrocks/qenv": "^4.0.6",
-    "@pushrocks/tapbundle": "^3.0.7",
+    "@pushrocks/tapbundle": "^3.0.13",
-    "@types/node": "^10.12.2",
+    "@types/node": "^12.12.5",
-    "tslint": "^5.11.0",
+    "tslint": "^5.20.0",
-    "tslint-config-prettier": "^1.15.0"
+    "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "@pushrocks/lik": "^3.0.1",
+    "@pushrocks/lik": "^3.0.11",
-    "@pushrocks/smartdelay": "^2.0.2",
+    "@pushrocks/smartdelay": "^2.0.3",
-    "@pushrocks/smartlog-interfaces": "^2.0.2",
+    "@pushrocks/smartlog-interfaces": "^2.0.9",
-    "@pushrocks/smartpromise": "^2.0.5",
+    "@pushrocks/smartpromise": "^3.0.6",
-    "@types/elasticsearch": "^5.0.28",
+    "@pushrocks/smarttime": "^3.0.12",
-    "elasticsearch": "^15.2.0"
+    "@types/elasticsearch": "^5.0.35",
+    "elasticsearch": "^16.5.0"
   }
 }

test/test.ts

@@ -8,11 +8,11 @@ let testElasticLog: elasticsearch.ElasticSearch<any>;
 
 tap.test('first test', async () => {
   testElasticLog = new elasticsearch.ElasticSearch({
-    domain: process.env.ELK_DOMAIN,
+    indexPrefix: 'smartlog',
-    port: parseInt(process.env.ELK_PORT, 10),
+    indexRetention: 7,
-    ssl: true,
+    domain: testQenv.getEnvVarOnDemand('ELK_DOMAIN'),
-    user: process.env.ELK_USER,
+    port: parseInt(testQenv.getEnvVarOnDemand('ELK_PORT'), 10),
-    pass: process.env.ELK_PASS
+    ssl: true
   });
   expect(testElasticLog).to.be.instanceOf(elasticsearch.ElasticSearch);
 });

ts/elasticsearch.classes.elasticindex.ts

@@ -14,32 +14,93 @@ export class ElasticIndex {
 
   public async ensureIndex(indexArg: string) {
     const done = plugins.smartpromise.defer();
-    if(this.stringmap.checkString(indexArg)) {
+    if (this.stringmap.checkString(indexArg)) {
+      done.resolve();
       return;
     }
-    this.elasticSearchRef.client.cat.indices({
+    this.elasticSearchRef.client.cat.indices(
+      {
         format: 'json',
         bytes: 'm'
-    }, async (err, response: any[]) => {
+      },
-      // console.log(response);
+      async (err, responseArg: any[]) => {
-      const index = response.find(indexObject => {
+        if(err) {
+          console.log(err);
+          return;
+        }
+
+        // lets delete indexes that violate the retention
+        if(Array.isArray(responseArg)) {
+          const filteredIndices = responseArg.filter(indexObjectArg => {
+            return indexObjectArg.index.startsWith('smartlog');
+          });
+          const filteredIndexNames = filteredIndices.map(indexObjectArg => {
+            return indexObjectArg.index;
+          });
+          this.deleteOldIndices(filteredIndexNames);
+        }
+
+        let index = null;
+        
+        if(Array.isArray(responseArg)) {
+          index = responseArg.find(indexObject => {
             return indexObject.index === indexArg;
           });
+        }
 
-      if(!index) {
+        if (!index) {
           const done2 = plugins.smartpromise.defer();
-        this.elasticSearchRef.client.indices.create({
+          this.elasticSearchRef.client.indices.create(
-          waitForActiveShards: '2',
+            {
+              
+              waitForActiveShards: '1',
               index: indexArg
-        }, (error, response) => {
+            },
+            (error, response) => {
               // console.lof(response)
               done2.resolve();
-        });
+            }
+          );
           await done2.promise;
         }
         this.stringmap.addString(indexArg);
         done.resolve();
-    });
+      }
+    );
     await done.promise;
   }
+
+  public createNewIndex(indexNameArg: string) {
+
+  }
+
+  public async deleteOldIndices(indicesArray: string[]) {
+    const todayAsUnix: number = Date.now();
+        const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
+          this.elasticSearchRef.indexRetention
+        );
+    for (const indexName of indicesArray) {
+      const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName);
+      const dateAsUnix: number = new Date(
+        `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
+      ).getTime();
+      if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
+        console.log(`found old index ${indexName}`);
+        const done2 = plugins.smartpromise.defer();
+        this.elasticSearchRef.client.indices.delete(
+          {
+            index: indexName
+          },
+          (err2, response2) => {
+            if (err2) {
+              console.log(err2);
+            }
+            console.log(`deleted ${indexName}`);
+            done2.resolve();
+          }
+        );
+        await done2.promise;
+      }
+    }
+  }
 }

ts/elasticsearch.classes.elasticsearch.ts

@@ -11,7 +11,9 @@ export interface IStandardLogParams {
   severity: string;
 }
 
-export interface IElasticLogConstructorOptions {
+export interface IElasticSearchConstructorOptions {
+  indexPrefix: string;
+  indexRetention: number;
   port: number;
   domain: string;
   ssl: boolean;
@@ -24,22 +26,27 @@ export class ElasticSearch<T> {
   public elasticScheduler = new ElasticScheduler(this);
   public elasticIndex: ElasticIndex = new ElasticIndex(this);
 
+  public indexPrefix: string;
+  public indexRetention: number;
+
   /**
    * sets up an instance of Elastic log
    * @param optionsArg
    */
-  constructor(optionsArg: IElasticLogConstructorOptions) {
+  constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
       host: this.computeHostString(optionsArg),
-      log: 'trace'
+      // log: 'trace'
     });
+    this.indexPrefix = optionsArg.indexPrefix;
+    this.indexRetention = optionsArg.indexRetention;
   }
 
   /**
    * computes the host string from the constructor options
    * @param optionsArg
    */
-  private computeHostString(optionsArg: IElasticLogConstructorOptions): string {
+  private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
     let hostString = `${optionsArg.domain}:${optionsArg.port}`;
     if (optionsArg.user && optionsArg.pass) {
       hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
@@ -49,15 +56,16 @@ export class ElasticSearch<T> {
     } else {
       hostString = `http://${hostString}`;
     }
+    console.log(hostString);
     return hostString;
   }
 
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
+    const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${(
-      '0' + now.getDate()
+      '0' +
-    ).slice(-2)}`;
+      (now.getMonth() + 1)
-
+    ).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
 
     if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
       this.elasticScheduler.scheduleDoc(logPackageArg);
@@ -81,17 +89,17 @@ export class ElasticSearch<T> {
           console.log(error);
           this.elasticScheduler.addFailedDoc(logPackageArg);
         } else {
-          console.log(`ElasticLog: ${logPackageArg.message}`);
+          // console.log(`ElasticLog: ${logPackageArg.message}`);
         }
       }
     );
   }
 
-  get logDestination (): ILogDestination {
+  get logDestination(): ILogDestination {
     return {
       handleLog: (smartlogPackageArg: ILogPackage) => {
         this.log(smartlogPackageArg);
       }
-    }
+    };
   }
 }

ts/elasticsearch.plugins.ts

@@ -1,6 +1,8 @@
 import * as elasticsearch from 'elasticsearch';
+import * as lik from '@pushrocks/lik';
 import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
 import * as smartpromise from '@pushrocks/smartpromise';
+import * as smarttime from '@pushrocks/smarttime';
 
-export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise };
+export { elasticsearch, lik, smartdelay, smartlogInterfaces, smartpromise, smarttime };