fix(core): update

This commit is contained in:
Philipp Kunz 2023-07-05 15:31:00 +02:00
parent 9c767d07e4
commit 60617f2fca
5 changed files with 58 additions and 83 deletions

View File

@ -1,4 +1,2 @@
required: required:
- ELK_NODE
- ELK_USER
- ELK_PASS

View File

@ -2,23 +2,21 @@ import { expect, tap } from '@pushrocks/tapbundle';
import { Qenv } from '@pushrocks/qenv'; import { Qenv } from '@pushrocks/qenv';
import * as elasticsearch from '../ts/index.js'; import * as elasticsearch from '../ts/index.js';
const testQenv = new Qenv('./', './.nogit/');
let testElasticLog: elasticsearch.ElasticSearch<any>; let testElasticLog: elasticsearch.ElasticSearch<any>;
tap.test('first test', async () => { tap.test('first test', async () => {
testElasticLog = new elasticsearch.ElasticSearch({ testElasticLog = new elasticsearch.ElasticSearch({
indexPrefix: 'smartlog', indexPrefix: 'testprefix',
indexRetention: 7, indexRetention: 7,
node: testQenv.getEnvVarOnDemand('ELK_NODE'), node: 'http://localhost:9200',
user: '', user: 'elastic',
pass: '', pass: 'YourPassword'
}); });
expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch); expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch);
}); });
tap.skip.test('should send a message to Elasticsearch', async () => { tap.test('should send a message to Elasticsearch', async () => {
testElasticLog.log({ await testElasticLog.log({
timestamp: Date.now(), timestamp: Date.now(),
type: 'increment', type: 'increment',
level: 'info', level: 'info',

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@apiclient.xyz/elasticsearch', name: '@apiclient.xyz/elasticsearch',
version: '1.0.41', version: '1.0.42',
description: 'log to elasticsearch in a kibana compatible format' description: 'log to elasticsearch in a kibana compatible format'
} }

View File

@ -12,91 +12,74 @@ export class ElasticIndex {
this.elasticSearchRef = elasticSearchInstanceArg; this.elasticSearchRef = elasticSearchInstanceArg;
} }
public async ensureIndex(indexArg: string) { public async ensureIndex(prefixArg: string, indexNameArg: string) {
const done = plugins.smartpromise.defer(); if (this.stringmap.checkString(indexNameArg)) {
if (this.stringmap.checkString(indexArg)) {
done.resolve();
return; return;
} }
this.elasticSearchRef.client.cat.indices( const responseArg = await this.elasticSearchRef.client.cat.indices({
{ format: 'json',
format: 'json', bytes: 'm',
bytes: 'm', }).catch(err => {
}, console.log(err);
async (err, responseArg: any[]) => { });
if (err) {
console.log(err);
return;
}
// lets delete indexes that violate the retention if (!responseArg) {
if (Array.isArray(responseArg)) { throw new Error('Could not get valid response from elastic search');
const filteredIndices = responseArg.filter((indexObjectArg) => { }
return indexObjectArg.index.startsWith('smartlog');
});
const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
return indexObjectArg.index;
});
this.deleteOldIndices(filteredIndexNames);
}
let index = null; // lets delete indexes that violate the retention
if (Array.isArray(responseArg.body)) {
const filteredIndices = responseArg.body.filter((indexObjectArg) => {
return indexObjectArg.index.startsWith(prefixArg);
});
const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
return indexObjectArg.index;
});
await this.deleteOldIndices(prefixArg, filteredIndexNames);
}
if (Array.isArray(responseArg)) { let index = null;
index = responseArg.find((indexObject) => {
return indexObject.index === indexArg;
});
}
if (!index) { if (Array.isArray(responseArg.body)) {
const done2 = plugins.smartpromise.defer(); index = responseArg.body.find((indexItemArg) => {
this.elasticSearchRef.client.indices.create( return indexItemArg.index === indexNameArg;
{ });
waitForActiveShards: '1', }
index: indexArg,
}, if (!index) {
(error, response) => { await this.createNewIndex(indexNameArg);
// console.lof(response) }
done2.resolve(); this.stringmap.addString(indexNameArg);
} return index;
);
await done2.promise;
}
this.stringmap.addString(indexArg);
done.resolve();
}
);
await done.promise;
} }
public createNewIndex(indexNameArg: string) {} public async createNewIndex(indexNameArg: string) {
const response = await this.elasticSearchRef.client.indices.create({
wait_for_active_shards: '1',
index: indexNameArg,
});
}
public async deleteOldIndices(indicesArray: string[]) { public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
const todayAsUnix: number = Date.now(); const todayAsUnix: number = Date.now();
const rententionPeriodAsUnix: number = plugins.smarttime.units.days( const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
this.elasticSearchRef.indexRetention this.elasticSearchRef.indexRetention
); );
for (const indexName of indicesArray) { for (const indexName of indicesArray) {
const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName); if (!indexName.startsWith(prefixArg)) continue;
const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`)
const regexResult = indexRegex.exec(indexName);
const dateAsUnix: number = new Date( const dateAsUnix: number = new Date(
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}` `${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
).getTime(); ).getTime();
if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) { if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
console.log(`found old index ${indexName}`); console.log(`found old index ${indexName}`);
const done2 = plugins.smartpromise.defer(); const response = await this.elasticSearchRef.client.indices.delete(
this.elasticSearchRef.client.indices.delete(
{ {
index: indexName, index: indexName,
}, }).catch(err => {
(err2, response2) => { console.log(err);
if (err2) { });
console.log(err2);
}
console.log(`deleted ${indexName}`);
done2.resolve();
}
);
await done2.promise;
} }
} }
} }

View File

@ -45,22 +45,18 @@ export class ElasticSearch<T> {
public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) { public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
const now = new Date(); const now = new Date();
const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${( const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`;
'0' +
(now.getMonth() + 1)
).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) { if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
this.elasticScheduler.scheduleDoc(logPackageArg); this.elasticScheduler.scheduleDoc(logPackageArg);
return; return;
} }
await this.elasticIndex.ensureIndex(indexToUse); const response = await this.elasticIndex.ensureIndex(this.indexPrefix, indexToUse);
console.log(response);
this.client.index( this.client.index(
{ {
index: indexToUse, index: indexToUse,
type: 'log',
body: { body: {
'@timestamp': new Date(logPackageArg.timestamp).toISOString(), '@timestamp': new Date(logPackageArg.timestamp).toISOString(),
...logPackageArg, ...logPackageArg,