From 60617f2fca6d441e82107aa88a91af12c81a7479 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Wed, 5 Jul 2023 15:31:00 +0200 Subject: [PATCH] fix(core): update --- qenv.yml | 4 +- test/test.nonci.ts | 14 ++- ts/00_commitinfo_data.ts | 2 +- ts/elasticsearch.classes.elasticindex.ts | 111 +++++++++------------- ts/elasticsearch.classes.elasticsearch.ts | 10 +- 5 files changed, 58 insertions(+), 83 deletions(-) diff --git a/qenv.yml b/qenv.yml index 59600f5..bd0ec3d 100644 --- a/qenv.yml +++ b/qenv.yml @@ -1,4 +1,2 @@ required: - - ELK_NODE - - ELK_USER - - ELK_PASS \ No newline at end of file + \ No newline at end of file diff --git a/test/test.nonci.ts b/test/test.nonci.ts index 2c2680b..e73becc 100644 --- a/test/test.nonci.ts +++ b/test/test.nonci.ts @@ -2,23 +2,21 @@ import { expect, tap } from '@pushrocks/tapbundle'; import { Qenv } from '@pushrocks/qenv'; import * as elasticsearch from '../ts/index.js'; -const testQenv = new Qenv('./', './.nogit/'); - let testElasticLog: elasticsearch.ElasticSearch; tap.test('first test', async () => { testElasticLog = new elasticsearch.ElasticSearch({ - indexPrefix: 'smartlog', + indexPrefix: 'testprefix', indexRetention: 7, - node: testQenv.getEnvVarOnDemand('ELK_NODE'), - user: '', - pass: '', + node: 'http://localhost:9200', + user: 'elastic', + pass: 'YourPassword' }); expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch); }); -tap.skip.test('should send a message to Elasticsearch', async () => { - testElasticLog.log({ +tap.test('should send a message to Elasticsearch', async () => { + await testElasticLog.log({ timestamp: Date.now(), type: 'increment', level: 'info', diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index e73a5a8..dfb498c 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/elasticsearch', - version: '1.0.41', + version: '1.0.42', description: 'log to elasticsearch in a kibana compatible format' } diff --git a/ts/elasticsearch.classes.elasticindex.ts b/ts/elasticsearch.classes.elasticindex.ts index 3b73b16..ec5b528 100644 --- a/ts/elasticsearch.classes.elasticindex.ts +++ b/ts/elasticsearch.classes.elasticindex.ts @@ -12,91 +12,74 @@ export class ElasticIndex { this.elasticSearchRef = elasticSearchInstanceArg; } - public async ensureIndex(indexArg: string) { - const done = plugins.smartpromise.defer(); - if (this.stringmap.checkString(indexArg)) { - done.resolve(); + public async ensureIndex(prefixArg: string, indexNameArg: string) { + if (this.stringmap.checkString(indexNameArg)) { return; } - this.elasticSearchRef.client.cat.indices( - { - format: 'json', - bytes: 'm', - }, - async (err, responseArg: any[]) => { - if (err) { - console.log(err); - return; - } + const responseArg = await this.elasticSearchRef.client.cat.indices({ + format: 'json', + bytes: 'm', + }).catch(err => { + console.log(err); + }); - // lets delete indexes that violate the retention - if (Array.isArray(responseArg)) { - const filteredIndices = responseArg.filter((indexObjectArg) => { - return indexObjectArg.index.startsWith('smartlog'); - }); - const filteredIndexNames = filteredIndices.map((indexObjectArg) => { - return indexObjectArg.index; - }); - this.deleteOldIndices(filteredIndexNames); - } + if (!responseArg) { + throw new Error('Could not get valid response from elastic search'); + } - let index = null; + // lets delete indexes that violate the retention + if (Array.isArray(responseArg.body)) { + const filteredIndices = responseArg.body.filter((indexObjectArg) => { + return indexObjectArg.index.startsWith(prefixArg); + }); + const filteredIndexNames = filteredIndices.map((indexObjectArg) => { + return indexObjectArg.index; + }); + await this.deleteOldIndices(prefixArg, filteredIndexNames); + } - if (Array.isArray(responseArg)) { - index = responseArg.find((indexObject) => { - return indexObject.index === indexArg; - }); - } + let index = null; - if (!index) { - const done2 = plugins.smartpromise.defer(); - this.elasticSearchRef.client.indices.create( - { - waitForActiveShards: '1', - index: indexArg, - }, - (error, response) => { - // console.lof(response) - done2.resolve(); - } - ); - await done2.promise; - } - this.stringmap.addString(indexArg); - done.resolve(); - } - ); - await done.promise; + if (Array.isArray(responseArg.body)) { + index = responseArg.body.find((indexItemArg) => { + return indexItemArg.index === indexNameArg; + }); + } + + if (!index) { + await this.createNewIndex(indexNameArg); + } + this.stringmap.addString(indexNameArg); + return index; } - public createNewIndex(indexNameArg: string) {} + public async createNewIndex(indexNameArg: string) { + const response = await this.elasticSearchRef.client.indices.create({ + wait_for_active_shards: '1', + index: indexNameArg, + }); + } - public async deleteOldIndices(indicesArray: string[]) { + public async deleteOldIndices(prefixArg: string, indicesArray: string[]) { const todayAsUnix: number = Date.now(); const rententionPeriodAsUnix: number = plugins.smarttime.units.days( this.elasticSearchRef.indexRetention ); for (const indexName of indicesArray) { - const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName); + if (!indexName.startsWith(prefixArg)) continue; + const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`) + const regexResult = indexRegex.exec(indexName); const dateAsUnix: number = new Date( `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}` ).getTime(); if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) { console.log(`found old index ${indexName}`); - const done2 = plugins.smartpromise.defer(); - this.elasticSearchRef.client.indices.delete( + const response = await this.elasticSearchRef.client.indices.delete( { index: indexName, - }, - (err2, response2) => { - if (err2) { - console.log(err2); - } - console.log(`deleted ${indexName}`); - done2.resolve(); - } - ); - await done2.promise; + }).catch(err => { + console.log(err); + }); } } } diff --git a/ts/elasticsearch.classes.elasticsearch.ts b/ts/elasticsearch.classes.elasticsearch.ts index af0a58b..cc03d03 100644 --- a/ts/elasticsearch.classes.elasticsearch.ts +++ b/ts/elasticsearch.classes.elasticsearch.ts @@ -45,22 +45,18 @@ export class ElasticSearch { public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) { const now = new Date(); - const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${( - '0' + - (now.getMonth() + 1) - ).slice(-2)}.${('0' + now.getDate()).slice(-2)}`; + const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`; if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) { this.elasticScheduler.scheduleDoc(logPackageArg); return; } - await this.elasticIndex.ensureIndex(indexToUse); - + const response = await this.elasticIndex.ensureIndex(this.indexPrefix, indexToUse); + console.log(response); this.client.index( { index: indexToUse, - type: 'log', body: { '@timestamp': new Date(logPackageArg.timestamp).toISOString(), ...logPackageArg,