From 30cbc05aa223f3dfe687c4fb7579fefed6c2b44e Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Tue, 1 Aug 2023 12:24:22 +0200 Subject: [PATCH] fix(core): update --- test/00tapwrap.ts | 0 test/test.nonci.ts | 6 +-- ts/00_commitinfo_data.ts | 2 +- ts/elasticsearch.classes.elasticindex.ts | 27 +++++++++++--- ts/elasticsearch.classes.elasticscheduler.ts | 37 ++++++++++++++----- ....ts => els.classes.smartlogdestination.ts} | 30 +++++++++++++-- ts/index.ts | 2 +- 7 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 test/00tapwrap.ts rename ts/{elasticsearch.classes.elasticsearch.ts => els.classes.smartlogdestination.ts} (74%) diff --git a/test/00tapwrap.ts b/test/00tapwrap.ts new file mode 100644 index 0000000..e69de29 diff --git a/test/test.nonci.ts b/test/test.nonci.ts index 6d3eff7..f3f5d13 100644 --- a/test/test.nonci.ts +++ b/test/test.nonci.ts @@ -2,10 +2,10 @@ import { expect, tap } from '@pushrocks/tapbundle'; import { Qenv } from '@pushrocks/qenv'; import * as elasticsearch from '../ts/index.js'; -let testElasticLog: elasticsearch.ElasticSearch; +let testElasticLog: elasticsearch.ElsSmartlogDestination; tap.test('first test', async () => { - testElasticLog = new elasticsearch.ElasticSearch({ + testElasticLog = new elasticsearch.ElsSmartlogDestination({ indexPrefix: 'testprefix', indexRetention: 7, node: 'http://localhost:9200', @@ -14,7 +14,7 @@ tap.test('first test', async () => { password: 'YourPassword' } }); - expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch); + expect(testElasticLog).toBeInstanceOf(elasticsearch.ElsSmartlogDestination); }); tap.test('should send a message to Elasticsearch', async () => { diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 71b4497..23636e3 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/elasticsearch', - version: '1.0.44', + version: '1.0.45', description: 'log to elasticsearch in a kibana compatible format' } diff --git a/ts/elasticsearch.classes.elasticindex.ts b/ts/elasticsearch.classes.elasticindex.ts index b4ed17d..cc06a88 100644 --- a/ts/elasticsearch.classes.elasticindex.ts +++ b/ts/elasticsearch.classes.elasticindex.ts @@ -1,14 +1,13 @@ import * as plugins from './elasticsearch.plugins.js'; -import { ElasticSearch } from './elasticsearch.classes.elasticsearch.js'; +import { ElsSmartlogDestination } from './els.classes.smartlogdestination.js'; import { type ILogPackage } from '@pushrocks/smartlog-interfaces'; - import { Stringmap } from '@pushrocks/lik'; export class ElasticIndex { private stringmap = new Stringmap(); - private elasticSearchRef: ElasticSearch; + private elasticSearchRef: ElsSmartlogDestination; - constructor(elasticSearchInstanceArg: ElasticSearch) { + constructor(elasticSearchInstanceArg: ElsSmartlogDestination) { this.elasticSearchRef = elasticSearchInstanceArg; } @@ -16,6 +15,7 @@ export class ElasticIndex { if (this.stringmap.checkString(indexNameArg)) { return indexNameArg; } + const responseArg = await this.elasticSearchRef.client.cat.indices({ format: 'json', bytes: 'm', @@ -27,7 +27,6 @@ export class ElasticIndex { throw new Error('Could not get valid response from elastic search'); } - // lets delete indexes that violate the retention if (Array.isArray(responseArg.body)) { const filteredIndices = responseArg.body.filter((indexObjectArg) => { return indexObjectArg.index.startsWith(prefixArg); @@ -49,6 +48,7 @@ export class ElasticIndex { if (!index) { await this.createNewIndex(indexNameArg); } + this.stringmap.addString(indexNameArg); return index; } @@ -57,6 +57,23 @@ export class ElasticIndex { const response = await this.elasticSearchRef.client.indices.create({ wait_for_active_shards: '1', index: indexNameArg, + body: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + logPackageArg: { + properties: { + payload: { + type: 'object', + dynamic: true + } + } + }, + }, + }, + }, }); } diff --git a/ts/elasticsearch.classes.elasticscheduler.ts b/ts/elasticsearch.classes.elasticscheduler.ts index fe18964..aa7ac54 100644 --- a/ts/elasticsearch.classes.elasticscheduler.ts +++ b/ts/elasticsearch.classes.elasticscheduler.ts @@ -1,29 +1,48 @@ -import { ElasticSearch, type IStandardLogParams } from './elasticsearch.classes.elasticsearch.js'; +import { ElsSmartlogDestination, type IStandardLogParams } from './els.classes.smartlogdestination.js'; export class ElasticScheduler { - elasticSearchRef: ElasticSearch; + elasticSearchRef: ElsSmartlogDestination; docsScheduled = false; docsStorage: any[] = []; - constructor(elasticLogRefArg: ElasticSearch) { + // maximum size of the buffer + maxBufferSize = 500; + + constructor(elasticLogRefArg: ElsSmartlogDestination) { this.elasticSearchRef = elasticLogRefArg; } public addFailedDoc(objectArg: any | IStandardLogParams) { - this.docsStorage.push(objectArg); + this.addToStorage(objectArg); this.setRetry(); } + public scheduleDoc(logObject: any) { + this.addToStorage(logObject); + } + + private addToStorage(logObject: any) { this.docsStorage.push(logObject); + + // if buffer is full, send logs immediately + if (this.docsStorage.length >= this.maxBufferSize) { + this.flushLogsToElasticSearch(); + } + } + + private flushLogsToElasticSearch() { + const oldStorage = this.docsStorage; + this.docsStorage = []; + + for (let logObject of oldStorage) { + this.elasticSearchRef.log(logObject, true); + } } public setRetry() { setTimeout(() => { - const oldStorage = this.docsStorage; - this.docsStorage = []; - for (let logObject of oldStorage) { - this.elasticSearchRef.log(logObject, true); - } + this.flushLogsToElasticSearch(); + if (this.docsStorage.length === 0) { console.log('ElasticLog retry success!!!'); this.docsScheduled = false; diff --git a/ts/elasticsearch.classes.elasticsearch.ts b/ts/els.classes.smartlogdestination.ts similarity index 74% rename from ts/elasticsearch.classes.elasticsearch.ts rename to ts/els.classes.smartlogdestination.ts index 91f57c7..b318169 100644 --- a/ts/elasticsearch.classes.elasticsearch.ts +++ b/ts/els.classes.smartlogdestination.ts @@ -21,7 +21,7 @@ export interface IElasticSearchConstructorOptions { }; } -export class ElasticSearch { +export class ElsSmartlogDestination { public client: ElasticClient; public elasticScheduler = new ElasticScheduler(this); public elasticIndex: ElasticIndex = new ElasticIndex(this); @@ -38,7 +38,7 @@ export class ElasticSearch { node: optionsArg.node, ...(optionsArg.auth && { auth: optionsArg.auth }), }); - this.indexPrefix = optionsArg.indexPrefix; + this.indexPrefix = `${optionsArg.indexPrefix}`; this.indexRetention = optionsArg.indexRetention; } @@ -51,7 +51,31 @@ export class ElasticSearch { return; } - const resultIndexName = await this.elasticIndex.ensureIndex(this.indexPrefix, indexToUse); + // Make sure the index is created with a mapping for dynamic JSON + const indexExists = await this.client.indices.exists({ index: indexToUse }); + if (!indexExists.body) { + await this.client.indices.create({ + index: indexToUse, + body: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + logPackageArg: { + properties: { + payload: { + type: 'object', + dynamic: true + } + } + }, + }, + }, + }, + }); + } + this.client.index( { index: indexToUse, diff --git a/ts/index.ts b/ts/index.ts index e94a7ea..982d63b 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -1 +1 @@ -export * from './elasticsearch.classes.elasticsearch.js'; +export * from './els.classes.smartlogdestination.js';