diff --git a/package-lock.json b/package-lock.json index a7838c8..e9bbd1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -198,6 +198,16 @@ "which": "^1.3.1" } }, + "@pushrocks/smarttime": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@pushrocks/smarttime/-/smarttime-3.0.2.tgz", + "integrity": "sha512-tFo2GZDwNa14umK3fNglvj9G/S981TSS1VZAMlIw7z43uMysbbJtjb8qpo625CilxEbi7D+RcyW5PJbCH2Wf6g==", + "requires": { + "@pushrocks/smartpromise": "^2.0.5", + "@types/luxon": "^1.4.0", + "luxon": "^1.4.3" + } + }, "@pushrocks/tapbundle": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@pushrocks/tapbundle/-/tapbundle-3.0.7.tgz", @@ -260,6 +270,11 @@ "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz", "integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw==" }, + "@types/luxon": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-1.4.1.tgz", + "integrity": "sha512-mYv/gbkOJ40CDgR8st5sosfFNrJncdlkpdzQSNRdU86UQg3oWWmll4AO/7B8F5FlBC6YrIXqXDSnkoCBqo+uMA==" + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -717,6 +732,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, + "luxon": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.5.0.tgz", + "integrity": "sha512-9IfJ0a5jA3poJSQnamaoOdPG0ulYk1JUI5IYmEdUhUc8dto4t6TVTu+oIM7fpSYy01GwUAA8kGUZMVe7Qx9Y+Q==" + }, "make-error": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", diff --git a/package.json b/package.json index 2bb8420..c352c1b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@pushrocks/smartdelay": "^2.0.2", "@pushrocks/smartlog-interfaces": "^2.0.2", "@pushrocks/smartpromise": "^2.0.5", + "@pushrocks/smarttime": "^3.0.2", "@types/elasticsearch": "^5.0.28", "elasticsearch": "^15.2.0" } diff --git a/test/test.ts b/test/test.ts index 830b2e7..ef76713 100644 --- a/test/test.ts +++ b/test/test.ts @@ -8,6 +8,8 @@ let testElasticLog: elasticsearch.ElasticSearch; tap.test('first test', async () => { testElasticLog = new elasticsearch.ElasticSearch({ + indexPrefix: 'smartlog', + indexRetention: 7, domain: process.env.ELK_DOMAIN, port: parseInt(process.env.ELK_PORT, 10), ssl: true, diff --git a/ts/elasticsearch.classes.elasticindex.ts b/ts/elasticsearch.classes.elasticindex.ts index 766709e..1dc014e 100644 --- a/ts/elasticsearch.classes.elasticindex.ts +++ b/ts/elasticsearch.classes.elasticindex.ts @@ -14,32 +14,68 @@ export class ElasticIndex { public async ensureIndex(indexArg: string) { const done = plugins.smartpromise.defer(); - if(this.stringmap.checkString(indexArg)) { + if (this.stringmap.checkString(indexArg)) { return; } - this.elasticSearchRef.client.cat.indices({ - format: 'json', - bytes: 'm' - }, async (err, response: any[]) => { - // console.log(response); - const index = response.find(indexObject => { - return indexObject.index === indexArg; - }); - - if(!index) { - const done2 = plugins.smartpromise.defer(); - this.elasticSearchRef.client.indices.create({ - waitForActiveShards: '2', - index: indexArg - }, (error, response) => { - // console.lof(response) - done2.resolve(); + this.elasticSearchRef.client.cat.indices( + { + format: 'json', + bytes: 'm' + }, + async (err, response: any[]) => { + // lets delete indexes that violate the retention + const filteredIndices = response.filter(indexObjectArg => { + return indexObjectArg.index.startsWith('smartlog'); }); - await done2.promise; + const filteredIndexNames = filteredIndices.map(indexObjectArg => { + return indexObjectArg.name; + }); + const todayAsUnix: number = Date.now(); + const rententionPeriodAsUnix: number = plugins.smarttime.units.days( + this.elasticSearchRef.indexRetention + ); + for (const indexName of filteredIndexNames) { + const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/; + const dateAsUnix: number = new Date( + `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}` + ).getTime(); + if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) { + const done2 = plugins.smartpromise.defer(); + this.elasticSearchRef.client.indices.delete({ + index: indexName + }, (err2, response2) => { + if(err2) { + console.log(err2); + } + done.resolve(); + }); + await done2.promise; + } + } + + // console.log(response); + const index = response.find(indexObject => { + return indexObject.index === indexArg; + }); + + if (!index) { + const done2 = plugins.smartpromise.defer(); + this.elasticSearchRef.client.indices.create( + { + waitForActiveShards: '2', + index: indexArg + }, + (error, response) => { + // console.lof(response) + done2.resolve(); + } + ); + await done2.promise; + } + this.stringmap.addString(indexArg); + done.resolve(); } - this.stringmap.addString(indexArg); - done.resolve(); - }); + ); await done.promise; } } diff --git a/ts/elasticsearch.classes.elasticsearch.ts b/ts/elasticsearch.classes.elasticsearch.ts index 618b5d0..a781e10 100644 --- a/ts/elasticsearch.classes.elasticsearch.ts +++ b/ts/elasticsearch.classes.elasticsearch.ts @@ -11,7 +11,9 @@ export interface IStandardLogParams { severity: string; } -export interface IElasticLogConstructorOptions { +export interface IElasticSearchConstructorOptions { + indexPrefix: string; + indexRetention: number; port: number; domain: string; ssl: boolean; @@ -24,22 +26,27 @@ export class ElasticSearch { public elasticScheduler = new ElasticScheduler(this); public elasticIndex: ElasticIndex = new ElasticIndex(this); + public indexPrefix: string; + public indexRetention: number; + /** * sets up an instance of Elastic log * @param optionsArg */ - constructor(optionsArg: IElasticLogConstructorOptions) { + constructor(optionsArg: IElasticSearchConstructorOptions) { this.client = new ElasticClient({ host: this.computeHostString(optionsArg), log: 'trace' }); + this.indexPrefix = optionsArg.indexPrefix; + this.indexRetention = optionsArg.indexRetention; } /** * computes the host string from the constructor options * @param optionsArg */ - private computeHostString(optionsArg: IElasticLogConstructorOptions): string { + private computeHostString(optionsArg: IElasticSearchConstructorOptions): string { let hostString = `${optionsArg.domain}:${optionsArg.port}`; if (optionsArg.user && optionsArg.pass) { hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`; @@ -54,7 +61,7 @@ export class ElasticSearch { public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) { const now = new Date(); - const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${( + const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${( '0' + now.getDate() ).slice(-2)}`; diff --git a/ts/elasticsearch.plugins.ts b/ts/elasticsearch.plugins.ts index 8ae4f66..6aff9a2 100644 --- a/ts/elasticsearch.plugins.ts +++ b/ts/elasticsearch.plugins.ts @@ -2,5 +2,6 @@ import * as elasticsearch from 'elasticsearch'; import * as smartdelay from '@pushrocks/smartdelay'; import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces'; import * as smartpromise from '@pushrocks/smartpromise'; +import * as smarttime from '@pushrocks/smarttime'; -export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise }; +export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise, smarttime };