fix(clean up old indices): update

package-lock.json

@@ -198,6 +198,16 @@
         "which": "^1.3.1"
       }
     },
+    "@pushrocks/smarttime": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@pushrocks/smarttime/-/smarttime-3.0.2.tgz",
+      "integrity": "sha512-tFo2GZDwNa14umK3fNglvj9G/S981TSS1VZAMlIw7z43uMysbbJtjb8qpo625CilxEbi7D+RcyW5PJbCH2Wf6g==",
+      "requires": {
+        "@pushrocks/smartpromise": "^2.0.5",
+        "@types/luxon": "^1.4.0",
+        "luxon": "^1.4.3"
+      }
+    },
     "@pushrocks/tapbundle": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/@pushrocks/tapbundle/-/tapbundle-3.0.7.tgz",
@@ -260,6 +270,11 @@
       "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz",
       "integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw=="
     },
+    "@types/luxon": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-1.4.1.tgz",
+      "integrity": "sha512-mYv/gbkOJ40CDgR8st5sosfFNrJncdlkpdzQSNRdU86UQg3oWWmll4AO/7B8F5FlBC6YrIXqXDSnkoCBqo+uMA=="
+    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -717,6 +732,11 @@
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
       "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
     },
+    "luxon": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.5.0.tgz",
+      "integrity": "sha512-9IfJ0a5jA3poJSQnamaoOdPG0ulYk1JUI5IYmEdUhUc8dto4t6TVTu+oIM7fpSYy01GwUAA8kGUZMVe7Qx9Y+Q=="
+    },
     "make-error": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",

package.json

@@ -27,6 +27,7 @@
     "@pushrocks/smartdelay": "^2.0.2",
     "@pushrocks/smartlog-interfaces": "^2.0.2",
     "@pushrocks/smartpromise": "^2.0.5",
+    "@pushrocks/smarttime": "^3.0.2",
     "@types/elasticsearch": "^5.0.28",
     "elasticsearch": "^15.2.0"
   }

test/test.ts

@@ -8,6 +8,8 @@ let testElasticLog: elasticsearch.ElasticSearch<any>;
 
 tap.test('first test', async () => {
   testElasticLog = new elasticsearch.ElasticSearch({
+    indexPrefix: 'smartlog',
+    indexRetention: 7,
     domain: process.env.ELK_DOMAIN,
     port: parseInt(process.env.ELK_PORT, 10),
     ssl: true,

ts/elasticsearch.classes.elasticindex.ts

@@ -14,32 +14,68 @@ export class ElasticIndex {
 
   public async ensureIndex(indexArg: string) {
     const done = plugins.smartpromise.defer();
-    if(this.stringmap.checkString(indexArg)) {
+    if (this.stringmap.checkString(indexArg)) {
       return;
     }
-    this.elasticSearchRef.client.cat.indices({
-      format: 'json',
-      bytes: 'm'
-    }, async (err, response: any[]) => {
-      // console.log(response);
-      const index = response.find(indexObject => {
-        return indexObject.index === indexArg;
-      });
-
-      if(!index) {
-        const done2 = plugins.smartpromise.defer();
-        this.elasticSearchRef.client.indices.create({
-          waitForActiveShards: '2',
-          index: indexArg
-        }, (error, response) => {
-          // console.lof(response)
-          done2.resolve();
+    this.elasticSearchRef.client.cat.indices(
+      {
+        format: 'json',
+        bytes: 'm'
+      },
+      async (err, response: any[]) => {
+        // lets delete indexes that violate the retention
+        const filteredIndices = response.filter(indexObjectArg => {
+          return indexObjectArg.index.startsWith('smartlog');
         });
-        await done2.promise;
+        const filteredIndexNames = filteredIndices.map(indexObjectArg => {
+          return indexObjectArg.name;
+        });
+        const todayAsUnix: number = Date.now();
+        const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
+          this.elasticSearchRef.indexRetention
+        );
+        for (const indexName of filteredIndexNames) {
+          const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/;
+          const dateAsUnix: number = new Date(
+            `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
+          ).getTime();
+          if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
+            const done2 = plugins.smartpromise.defer();
+            this.elasticSearchRef.client.indices.delete({
+              index: indexName
+            }, (err2, response2) => {
+              if(err2) {
+                console.log(err2);
+              }
+              done.resolve();
+            });
+            await done2.promise;
+          }
+        }
+
+        // console.log(response);
+        const index = response.find(indexObject => {
+          return indexObject.index === indexArg;
+        });
+
+        if (!index) {
+          const done2 = plugins.smartpromise.defer();
+          this.elasticSearchRef.client.indices.create(
+            {
+              waitForActiveShards: '2',
+              index: indexArg
+            },
+            (error, response) => {
+              // console.lof(response)
+              done2.resolve();
+            }
+          );
+          await done2.promise;
+        }
+        this.stringmap.addString(indexArg);
+        done.resolve();
       }
-      this.stringmap.addString(indexArg);
-      done.resolve();
-    });
+    );
     await done.promise;
   }
 }

ts/elasticsearch.classes.elasticsearch.ts

@@ -11,7 +11,9 @@ export interface IStandardLogParams {
   severity: string;
 }
 
-export interface IElasticLogConstructorOptions {
+export interface IElasticSearchConstructorOptions {
+  indexPrefix: string;
+  indexRetention: number;
   port: number;
   domain: string;
   ssl: boolean;
@@ -24,22 +26,27 @@ export class ElasticSearch<T> {
   public elasticScheduler = new ElasticScheduler(this);
   public elasticIndex: ElasticIndex = new ElasticIndex(this);
 
+  public indexPrefix: string;
+  public indexRetention: number;
+
   /**
    * sets up an instance of Elastic log
    * @param optionsArg
    */
-  constructor(optionsArg: IElasticLogConstructorOptions) {
+  constructor(optionsArg: IElasticSearchConstructorOptions) {
     this.client = new ElasticClient({
       host: this.computeHostString(optionsArg),
       log: 'trace'
     });
+    this.indexPrefix = optionsArg.indexPrefix;
+    this.indexRetention = optionsArg.indexRetention;
   }
 
   /**
    * computes the host string from the constructor options
    * @param optionsArg
    */
-  private computeHostString(optionsArg: IElasticLogConstructorOptions): string {
+  private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
     let hostString = `${optionsArg.domain}:${optionsArg.port}`;
     if (optionsArg.user && optionsArg.pass) {
       hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
@@ -54,7 +61,7 @@ export class ElasticSearch<T> {
 
   public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
     const now = new Date();
-    const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
+    const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
       '0' + now.getDate()
     ).slice(-2)}`;
 

ts/elasticsearch.plugins.ts

@@ -2,5 +2,6 @@ import * as elasticsearch from 'elasticsearch';
 import * as smartdelay from '@pushrocks/smartdelay';
 import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
 import * as smartpromise from '@pushrocks/smartpromise';
+import * as smarttime from '@pushrocks/smarttime';
 
-export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise };
+export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise, smarttime };