fix(core): update
This commit is contained in:
parent
9c767d07e4
commit
60617f2fca
@ -2,23 +2,21 @@ import { expect, tap } from '@pushrocks/tapbundle';
|
||||
import { Qenv } from '@pushrocks/qenv';
|
||||
import * as elasticsearch from '../ts/index.js';
|
||||
|
||||
const testQenv = new Qenv('./', './.nogit/');
|
||||
|
||||
let testElasticLog: elasticsearch.ElasticSearch<any>;
|
||||
|
||||
tap.test('first test', async () => {
|
||||
testElasticLog = new elasticsearch.ElasticSearch({
|
||||
indexPrefix: 'smartlog',
|
||||
indexPrefix: 'testprefix',
|
||||
indexRetention: 7,
|
||||
node: testQenv.getEnvVarOnDemand('ELK_NODE'),
|
||||
user: '',
|
||||
pass: '',
|
||||
node: 'http://localhost:9200',
|
||||
user: 'elastic',
|
||||
pass: 'YourPassword'
|
||||
});
|
||||
expect(testElasticLog).toBeInstanceOf(elasticsearch.ElasticSearch);
|
||||
});
|
||||
|
||||
tap.skip.test('should send a message to Elasticsearch', async () => {
|
||||
testElasticLog.log({
|
||||
tap.test('should send a message to Elasticsearch', async () => {
|
||||
await testElasticLog.log({
|
||||
timestamp: Date.now(),
|
||||
type: 'increment',
|
||||
level: 'info',
|
||||
|
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@apiclient.xyz/elasticsearch',
|
||||
version: '1.0.41',
|
||||
version: '1.0.42',
|
||||
description: 'log to elasticsearch in a kibana compatible format'
|
||||
}
|
||||
|
@ -12,91 +12,74 @@ export class ElasticIndex {
|
||||
this.elasticSearchRef = elasticSearchInstanceArg;
|
||||
}
|
||||
|
||||
public async ensureIndex(indexArg: string) {
|
||||
const done = plugins.smartpromise.defer();
|
||||
if (this.stringmap.checkString(indexArg)) {
|
||||
done.resolve();
|
||||
public async ensureIndex(prefixArg: string, indexNameArg: string) {
|
||||
if (this.stringmap.checkString(indexNameArg)) {
|
||||
return;
|
||||
}
|
||||
this.elasticSearchRef.client.cat.indices(
|
||||
{
|
||||
format: 'json',
|
||||
bytes: 'm',
|
||||
},
|
||||
async (err, responseArg: any[]) => {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
return;
|
||||
}
|
||||
const responseArg = await this.elasticSearchRef.client.cat.indices({
|
||||
format: 'json',
|
||||
bytes: 'm',
|
||||
}).catch(err => {
|
||||
console.log(err);
|
||||
});
|
||||
|
||||
// lets delete indexes that violate the retention
|
||||
if (Array.isArray(responseArg)) {
|
||||
const filteredIndices = responseArg.filter((indexObjectArg) => {
|
||||
return indexObjectArg.index.startsWith('smartlog');
|
||||
});
|
||||
const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
|
||||
return indexObjectArg.index;
|
||||
});
|
||||
this.deleteOldIndices(filteredIndexNames);
|
||||
}
|
||||
if (!responseArg) {
|
||||
throw new Error('Could not get valid response from elastic search');
|
||||
}
|
||||
|
||||
let index = null;
|
||||
// lets delete indexes that violate the retention
|
||||
if (Array.isArray(responseArg.body)) {
|
||||
const filteredIndices = responseArg.body.filter((indexObjectArg) => {
|
||||
return indexObjectArg.index.startsWith(prefixArg);
|
||||
});
|
||||
const filteredIndexNames = filteredIndices.map((indexObjectArg) => {
|
||||
return indexObjectArg.index;
|
||||
});
|
||||
await this.deleteOldIndices(prefixArg, filteredIndexNames);
|
||||
}
|
||||
|
||||
if (Array.isArray(responseArg)) {
|
||||
index = responseArg.find((indexObject) => {
|
||||
return indexObject.index === indexArg;
|
||||
});
|
||||
}
|
||||
let index = null;
|
||||
|
||||
if (!index) {
|
||||
const done2 = plugins.smartpromise.defer();
|
||||
this.elasticSearchRef.client.indices.create(
|
||||
{
|
||||
waitForActiveShards: '1',
|
||||
index: indexArg,
|
||||
},
|
||||
(error, response) => {
|
||||
// console.lof(response)
|
||||
done2.resolve();
|
||||
}
|
||||
);
|
||||
await done2.promise;
|
||||
}
|
||||
this.stringmap.addString(indexArg);
|
||||
done.resolve();
|
||||
}
|
||||
);
|
||||
await done.promise;
|
||||
if (Array.isArray(responseArg.body)) {
|
||||
index = responseArg.body.find((indexItemArg) => {
|
||||
return indexItemArg.index === indexNameArg;
|
||||
});
|
||||
}
|
||||
|
||||
if (!index) {
|
||||
await this.createNewIndex(indexNameArg);
|
||||
}
|
||||
this.stringmap.addString(indexNameArg);
|
||||
return index;
|
||||
}
|
||||
|
||||
public createNewIndex(indexNameArg: string) {}
|
||||
public async createNewIndex(indexNameArg: string) {
|
||||
const response = await this.elasticSearchRef.client.indices.create({
|
||||
wait_for_active_shards: '1',
|
||||
index: indexNameArg,
|
||||
});
|
||||
}
|
||||
|
||||
public async deleteOldIndices(indicesArray: string[]) {
|
||||
public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
|
||||
const todayAsUnix: number = Date.now();
|
||||
const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
|
||||
this.elasticSearchRef.indexRetention
|
||||
);
|
||||
for (const indexName of indicesArray) {
|
||||
const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/.exec(indexName);
|
||||
if (!indexName.startsWith(prefixArg)) continue;
|
||||
const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`)
|
||||
const regexResult = indexRegex.exec(indexName);
|
||||
const dateAsUnix: number = new Date(
|
||||
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
|
||||
).getTime();
|
||||
if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
|
||||
console.log(`found old index ${indexName}`);
|
||||
const done2 = plugins.smartpromise.defer();
|
||||
this.elasticSearchRef.client.indices.delete(
|
||||
const response = await this.elasticSearchRef.client.indices.delete(
|
||||
{
|
||||
index: indexName,
|
||||
},
|
||||
(err2, response2) => {
|
||||
if (err2) {
|
||||
console.log(err2);
|
||||
}
|
||||
console.log(`deleted ${indexName}`);
|
||||
done2.resolve();
|
||||
}
|
||||
);
|
||||
await done2.promise;
|
||||
}).catch(err => {
|
||||
console.log(err);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -45,22 +45,18 @@ export class ElasticSearch<T> {
|
||||
|
||||
public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
|
||||
const now = new Date();
|
||||
const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${(
|
||||
'0' +
|
||||
(now.getMonth() + 1)
|
||||
).slice(-2)}.${('0' + now.getDate()).slice(-2)}`;
|
||||
const indexToUse = `${this.indexPrefix}-${now.toISOString().split('T')[0]}`;
|
||||
|
||||
if (this.elasticScheduler.docsScheduled && !scheduleOverwrite) {
|
||||
this.elasticScheduler.scheduleDoc(logPackageArg);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.elasticIndex.ensureIndex(indexToUse);
|
||||
|
||||
const response = await this.elasticIndex.ensureIndex(this.indexPrefix, indexToUse);
|
||||
console.log(response);
|
||||
this.client.index(
|
||||
{
|
||||
index: indexToUse,
|
||||
type: 'log',
|
||||
body: {
|
||||
'@timestamp': new Date(logPackageArg.timestamp).toISOString(),
|
||||
...logPackageArg,
|
||||
|
Loading…
Reference in New Issue
Block a user