fix(core): update

This commit is contained in:
Philipp Kunz 2018-11-09 23:15:11 +01:00
parent aa43a221a0
commit f9866076ca
7 changed files with 141 additions and 60 deletions

36
package-lock.json generated
View File

@ -74,6 +74,19 @@
"@pushrocks/smartpromise": "^2.0.5" "@pushrocks/smartpromise": "^2.0.5"
} }
}, },
"@pushrocks/lik": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@pushrocks/lik/-/lik-3.0.1.tgz",
"integrity": "sha512-6dwRg7WMdY9drlqfo6NzOcw5ubVWYvIOSSMZz4Jsojpa8aDVwXd2VkD7sxO/PJRrbZzHkx/JXoY/epcWRjngUw==",
"requires": {
"@pushrocks/smartpromise": "^2.0.5",
"@types/lodash": "^4.14.112",
"@types/minimatch": "^3.0.3",
"lodash": "^4.17.10",
"minimatch": "^3.0.4",
"symbol-tree": "^3.2.2"
}
},
"@pushrocks/qenv": { "@pushrocks/qenv": {
"version": "2.0.2", "version": "2.0.2",
"resolved": "https://registry.npmjs.org/@pushrocks/qenv/-/qenv-2.0.2.tgz", "resolved": "https://registry.npmjs.org/@pushrocks/qenv/-/qenv-2.0.2.tgz",
@ -242,6 +255,16 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/lodash": {
"version": "4.14.118",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz",
"integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw=="
},
"@types/minimatch": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
"integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA=="
},
"@types/node": { "@types/node": {
"version": "10.12.2", "version": "10.12.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.2.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.2.tgz",
@ -328,8 +351,7 @@
"balanced-match": { "balanced-match": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
"dev": true
}, },
"bindings": { "bindings": {
"version": "1.3.0", "version": "1.3.0",
@ -341,7 +363,6 @@
"version": "1.1.11", "version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": { "requires": {
"balanced-match": "^1.0.0", "balanced-match": "^1.0.0",
"concat-map": "0.0.1" "concat-map": "0.0.1"
@ -468,8 +489,7 @@
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
"dev": true
}, },
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
@ -722,7 +742,6 @@
"version": "3.0.4", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": { "requires": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }
@ -940,6 +959,11 @@
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
}, },
"symbol-tree": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz",
"integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY="
},
"ts-node": { "ts-node": {
"version": "7.0.1", "version": "7.0.1",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-7.0.1.tgz",

View File

@ -23,8 +23,10 @@
"tslint-config-prettier": "^1.15.0" "tslint-config-prettier": "^1.15.0"
}, },
"dependencies": { "dependencies": {
"@pushrocks/lik": "^3.0.1",
"@pushrocks/smartdelay": "^2.0.2", "@pushrocks/smartdelay": "^2.0.2",
"@pushrocks/smartlog-interfaces": "^2.0.2", "@pushrocks/smartlog-interfaces": "^2.0.2",
"@pushrocks/smartpromise": "^2.0.5",
"@types/elasticsearch": "^5.0.28", "@types/elasticsearch": "^5.0.28",
"elasticsearch": "^15.2.0" "elasticsearch": "^15.2.0"
} }

View File

@ -0,0 +1,45 @@
import * as plugins from './elasticsearch.plugins';
import { ElasticSearch } from './elasticsearch.classes.elasticsearch';
import { ILogPackage } from '@pushrocks/smartlog-interfaces';
import { Stringmap } from '@pushrocks/lik';
export class ElasticIndex {
private stringmap = new Stringmap();
private elasticSearchRef: ElasticSearch<any>;
constructor(elasticSearchInstanceArg: ElasticSearch<ILogPackage>) {
this.elasticSearchRef = elasticSearchInstanceArg;
}
public async ensureIndex(indexArg: string) {
const done = plugins.smartpromise.defer();
if(this.stringmap.checkString(indexArg)) {
return;
}
this.elasticSearchRef.client.cat.indices({
format: 'json',
bytes: 'm'
}, async (err, response: any[]) => {
// console.log(response);
const index = response.find(indexObject => {
return indexObject.index === indexArg;
});
if(!index) {
const done2 = plugins.smartpromise.defer();
this.elasticSearchRef.client.indices.create({
waitForActiveShards: '2',
index: indexArg
}, (error, response) => {
// console.lof(response)
done2.resolve();
});
await done2.promise;
}
this.stringmap.addString(indexArg);
done.resolve();
});
await done.promise;
}
}

View File

@ -0,0 +1,44 @@
import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch';
export class ElasticScheduler {
elasticSearchRef: ElasticSearch<any>;
docsScheduled = false;
docsStorage: any[] = [];
constructor(elasticLogRefArg: ElasticSearch<any>) {
this.elasticSearchRef = elasticLogRefArg;
}
public addFailedDoc(objectArg: any | IStandardLogParams) {
this.docsStorage.push(objectArg);
this.setRetry();
}
public scheduleDoc(logObject: any) {
this.docsStorage.push(logObject);
}
public setRetry() {
setTimeout(() => {
const oldStorage = this.docsStorage;
this.docsStorage = [];
for (let logObject of oldStorage) {
this.elasticSearchRef.log(logObject, true);
}
if (this.docsStorage.length === 0) {
console.log('ElasticLog retry success!!!');
this.docsScheduled = false;
} else {
console.log('ElasticLog retry failed');
this.setRetry();
}
}, 5000);
}
public deferSend() {
if (!this.docsScheduled) {
console.log('Retry ElasticLog in 5 seconds!');
this.docsScheduled = true;
this.setRetry();
}
}
}

View File

@ -3,7 +3,8 @@ import { Client as ElasticClient } from 'elasticsearch';
import { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces'; import { ILogContext, ILogPackage, ILogDestination } from '@pushrocks/smartlog-interfaces';
// other classes // other classes
import { LogScheduler } from './elasticsearch.classes.logscheduler'; import { ElasticScheduler } from './elasticsearch.classes.elasticscheduler';
import { ElasticIndex } from './elasticsearch.classes.elasticindex';
export interface IStandardLogParams { export interface IStandardLogParams {
message: string; message: string;
@ -19,8 +20,9 @@ export interface IElasticLogConstructorOptions {
} }
export class ElasticSearch<T> { export class ElasticSearch<T> {
client: ElasticClient; public client: ElasticClient;
logScheduler = new LogScheduler(this); public elasticScheduler = new ElasticScheduler(this);
public elasticIndex: ElasticIndex = new ElasticIndex(this);
/** /**
* sets up an instance of Elastic log * sets up an instance of Elastic log
@ -52,15 +54,21 @@ export class ElasticSearch<T> {
public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) { public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
const now = new Date(); const now = new Date();
if (this.logScheduler.logsScheduled && !scheduleOverwrite) { const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
this.logScheduler.scheduleLog(logPackageArg); '0' + now.getDate()
).slice(-2)}`;
if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
this.elasticScheduler.scheduleDoc(logPackageArg);
return; return;
} }
await this.elasticIndex.ensureIndex(indexToUse);
this.client.index( this.client.index(
{ {
index: `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${( index: indexToUse,
'0' + now.getDate()
).slice(-2)}`,
type: 'log', type: 'log',
body: { body: {
'@timestamp': new Date(logPackageArg.timestamp).toISOString(), '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
@ -71,7 +79,7 @@ export class ElasticSearch<T> {
if (error) { if (error) {
console.log('ElasticLog encountered an error:'); console.log('ElasticLog encountered an error:');
console.log(error); console.log(error);
this.logScheduler.addFailedLog(logPackageArg); this.elasticScheduler.addFailedDoc(logPackageArg);
} else { } else {
console.log(`ElasticLog: ${logPackageArg.message}`); console.log(`ElasticLog: ${logPackageArg.message}`);
} }

View File

@ -1,44 +0,0 @@
import { ElasticSearch, IStandardLogParams } from './elasticsearch.classes.elasticsearch';
export class LogScheduler {
elasticLogRef: ElasticSearch<any>;
logsScheduled = false;
logStorage: any[] = [];
constructor(elasticLogRefArg: ElasticSearch<any>) {
this.elasticLogRef = elasticLogRefArg;
}
addFailedLog(objectArg: any | IStandardLogParams) {
this.logStorage.push(objectArg);
this.setRetry();
}
scheduleLog(logObject: any) {
this.logStorage.push(logObject);
}
setRetry() {
setTimeout(() => {
const oldStorage = this.logStorage;
this.logStorage = [];
for (let logObject of oldStorage) {
this.elasticLogRef.log(logObject, true);
}
if (this.logStorage.length === 0) {
console.log('ElasticLog retry success!!!');
this.logsScheduled = false;
} else {
console.log('ElasticLog retry failed');
this.setRetry();
}
}, 5000);
}
deferSend() {
if (!this.logsScheduled) {
console.log('Retry ElasticLog in 5 seconds!');
this.logsScheduled = true;
this.setRetry();
}
}
}

View File

@ -1,4 +1,6 @@
import * as elasticsearch from 'elasticsearch'; import * as elasticsearch from 'elasticsearch';
import * as smartdelay from '@pushrocks/smartdelay'; import * as smartdelay from '@pushrocks/smartdelay';
import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces'; import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
export { elasticsearch, smartdelay, smartlogInterfaces }; import * as smartpromise from '@pushrocks/smartpromise';
export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise };