From f9866076caae9395c5fb55102f9a5e799f1aa3cf Mon Sep 17 00:00:00 2001 From: Phil Kunz Date: Fri, 9 Nov 2018 23:15:11 +0100 Subject: [PATCH] fix(core): update --- package-lock.json | 36 +++++++++++++--- package.json | 2 + ts/elasticsearch.classes.elasticindex.ts | 45 ++++++++++++++++++++ ts/elasticsearch.classes.elasticscheduler.ts | 44 +++++++++++++++++++ ts/elasticsearch.classes.elasticsearch.ts | 26 +++++++---- ts/elasticsearch.classes.logscheduler.ts | 44 ------------------- ts/elasticsearch.plugins.ts | 4 +- 7 files changed, 141 insertions(+), 60 deletions(-) create mode 100644 ts/elasticsearch.classes.elasticindex.ts create mode 100644 ts/elasticsearch.classes.elasticscheduler.ts delete mode 100644 ts/elasticsearch.classes.logscheduler.ts diff --git a/package-lock.json b/package-lock.json index 4c6202c..b4face6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,6 +74,19 @@ "@pushrocks/smartpromise": "^2.0.5" } }, + "@pushrocks/lik": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@pushrocks/lik/-/lik-3.0.1.tgz", + "integrity": "sha512-6dwRg7WMdY9drlqfo6NzOcw5ubVWYvIOSSMZz4Jsojpa8aDVwXd2VkD7sxO/PJRrbZzHkx/JXoY/epcWRjngUw==", + "requires": { + "@pushrocks/smartpromise": "^2.0.5", + "@types/lodash": "^4.14.112", + "@types/minimatch": "^3.0.3", + "lodash": "^4.17.10", + "minimatch": "^3.0.4", + "symbol-tree": "^3.2.2" + } + }, "@pushrocks/qenv": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@pushrocks/qenv/-/qenv-2.0.2.tgz", @@ -242,6 +255,16 @@ "@types/node": "*" } }, + "@types/lodash": { + "version": "4.14.118", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz", + "integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw==" + }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" + }, "@types/node": { "version": "10.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.2.tgz", @@ -328,8 +351,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "bindings": { "version": "1.3.0", @@ -341,7 +363,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -468,8 +489,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "core-util-is": { "version": "1.0.2", @@ -722,7 +742,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -940,6 +959,11 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "symbol-tree": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz", + "integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY=" + }, "ts-node": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz", diff --git a/package.json b/package.json index 6dbb0ad..667094f 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,10 @@ "tslint-config-prettier": "^1.15.0" }, "dependencies": { + "@pushrocks/lik": "^3.0.1", "@pushrocks/smartdelay": "^2.0.2", "@pushrocks/smartlog-interfaces": "^2.0.2", + "@pushrocks/smartpromise": "^2.0.5", "@types/elasticsearch": "^5.0.28", "elasticsearch": "^15.2.0" } diff --git a/ts/elasticsearch.classes.elasticindex.ts b/ts/elasticsearch.classes.elasticindex.ts new file mode 100644 index 0000000..766709e --- /dev/null +++ b/ts/elasticsearch.classes.elasticindex.ts @@ -0,0 +1,45 @@ +import * as plugins from './elasticsearch.plugins'; +import { ElasticSearch } from './elasticsearch.classes.elasticsearch'; +import { ILogPackage } from '@pushrocks/smartlog-interfaces'; + +import { Stringmap } from '@pushrocks/lik'; + +export class ElasticIndex { + private stringmap = new Stringmap(); + private elasticSearchRef: ElasticSearch; + + constructor(elasticSearchInstanceArg: ElasticSearch) { + this.elasticSearchRef = elasticSearchInstanceArg; + } + + public async ensureIndex(indexArg: string) { + const done = plugins.smartpromise.defer(); + if(this.stringmap.checkString(indexArg)) { + return; + } + this.elasticSearchRef.client.cat.indices({ + format: 'json', + bytes: 'm' + }, async (err, response: any[]) => { + // console.log(response); + const index = response.find(indexObject => { + return indexObject.index === indexArg; + }); + + if(!index) { + const done2 = plugins.smartpromise.defer(); + this.elasticSearchRef.client.indices.create({ + waitForActiveShards: '2', + index: indexArg + }, (error, response) => { + // console.lof(response) + done2.resolve(); + }); + await done2.promise; + } + this.stringmap.addString(indexArg); + done.resolve(); + }); + await done.promise; + } +} diff --git a/ts/elasticsearch.classes.elasticscheduler.ts b/ts/elasticsearch.classes.elasticscheduler.ts new file mode 100644 index 0000000..5d09926 --- /dev/null +++ b/ts/elasticsearch.classes.elasticscheduler.ts @@ -0,0 +1,44 @@ +import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch'; + +export class ElasticScheduler { + elasticSearchRef: ElasticSearch; + docsScheduled = false; + docsStorage: any[] = []; + + constructor(elasticLogRefArg: ElasticSearch) { + this.elasticSearchRef = elasticLogRefArg; + } + + public addFailedDoc(objectArg: any | IStandardLogParams) { + this.docsStorage.push(objectArg); + this.setRetry(); + } + public scheduleDoc(logObject: any) { + this.docsStorage.push(logObject); + } + + public setRetry() { + setTimeout(() => { + const oldStorage = this.docsStorage; + this.docsStorage = []; + for (let logObject of oldStorage) { + this.elasticSearchRef.log(logObject, true); + } + if (this.docsStorage.length === 0) { + console.log('ElasticLog retry success!!!'); + this.docsScheduled = false; + } else { + console.log('ElasticLog retry failed'); + this.setRetry(); + } + }, 5000); + } + + public deferSend() { + if (!this.docsScheduled) { + console.log('Retry ElasticLog in 5 seconds!'); + this.docsScheduled = true; + this.setRetry(); + } + } +} diff --git a/ts/elasticsearch.classes.elasticsearch.ts b/ts/elasticsearch.classes.elasticsearch.ts index 7daddee..618b5d0 100644 --- a/ts/elasticsearch.classes.elasticsearch.ts +++ b/ts/elasticsearch.classes.elasticsearch.ts @@ -3,7 +3,8 @@ import { Client as ElasticClient } from 'elasticsearch'; import { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces'; // other classes -import { LogScheduler } from './elasticsearch.classes.logscheduler'; +import { ElasticScheduler } from './elasticsearch.classes.elasticscheduler'; +import { ElasticIndex } from './elasticsearch.classes.elasticindex'; export interface IStandardLogParams { message: string; @@ -19,8 +20,9 @@ export interface IElasticLogConstructorOptions { } export class ElasticSearch { - client: ElasticClient; - logScheduler = new LogScheduler(this); + public client: ElasticClient; + public elasticScheduler = new ElasticScheduler(this); + public elasticIndex: ElasticIndex = new ElasticIndex(this); /** * sets up an instance of Elastic log @@ -52,15 +54,21 @@ export class ElasticSearch { public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) { const now = new Date(); - if (this.logScheduler.logsScheduled && !scheduleOverwrite) { - this.logScheduler.scheduleLog(logPackageArg); + const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${( + '0' + now.getDate() + ).slice(-2)}`; + + + if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) { + this.elasticScheduler.scheduleDoc(logPackageArg); return; } + + await this.elasticIndex.ensureIndex(indexToUse); + this.client.index( { - index: `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${( - '0' + now.getDate() - ).slice(-2)}`, + index: indexToUse, type: 'log', body: { '@timestamp': new Date(logPackageArg.timestamp).toISOString(), @@ -71,7 +79,7 @@ export class ElasticSearch { if (error) { console.log('ElasticLog encountered an error:'); console.log(error); - this.logScheduler.addFailedLog(logPackageArg); + this.elasticScheduler.addFailedDoc(logPackageArg); } else { console.log(`ElasticLog: ${logPackageArg.message}`); } diff --git a/ts/elasticsearch.classes.logscheduler.ts b/ts/elasticsearch.classes.logscheduler.ts deleted file mode 100644 index 0d41f9a..0000000 --- a/ts/elasticsearch.classes.logscheduler.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch'; - -export class LogScheduler { - elasticLogRef: ElasticSearch; - logsScheduled = false; - logStorage: any[] = []; - - constructor(elasticLogRefArg: ElasticSearch) { - this.elasticLogRef = elasticLogRefArg; - } - - addFailedLog(objectArg: any | IStandardLogParams) { - this.logStorage.push(objectArg); - this.setRetry(); - } - scheduleLog(logObject: any) { - this.logStorage.push(logObject); - } - - setRetry() { - setTimeout(() => { - const oldStorage = this.logStorage; - this.logStorage = []; - for (let logObject of oldStorage) { - this.elasticLogRef.log(logObject, true); - } - if (this.logStorage.length === 0) { - console.log('ElasticLog retry success!!!'); - this.logsScheduled = false; - } else { - console.log('ElasticLog retry failed'); - this.setRetry(); - } - }, 5000); - } - - deferSend() { - if (!this.logsScheduled) { - console.log('Retry ElasticLog in 5 seconds!'); - this.logsScheduled = true; - this.setRetry(); - } - } -} diff --git a/ts/elasticsearch.plugins.ts b/ts/elasticsearch.plugins.ts index 5129bc5..8ae4f66 100644 --- a/ts/elasticsearch.plugins.ts +++ b/ts/elasticsearch.plugins.ts @@ -1,4 +1,6 @@ import * as elasticsearch from 'elasticsearch'; import * as smartdelay from '@pushrocks/smartdelay'; import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces'; -export { elasticsearch, smartdelay, smartlogInterfaces }; +import * as smartpromise from '@pushrocks/smartpromise'; + +export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise };