fix(clean up old indices): update
This commit is contained in:
parent
5f94db8d5d
commit
d636dab664
20
package-lock.json
generated
20
package-lock.json
generated
@ -198,6 +198,16 @@
|
|||||||
"which": "^1.3.1"
|
"which": "^1.3.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"@pushrocks/smarttime": {
|
||||||
|
"version": "3.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/@pushrocks/smarttime/-/smarttime-3.0.2.tgz",
|
||||||
|
"integrity": "sha512-tFo2GZDwNa14umK3fNglvj9G/S981TSS1VZAMlIw7z43uMysbbJtjb8qpo625CilxEbi7D+RcyW5PJbCH2Wf6g==",
|
||||||
|
"requires": {
|
||||||
|
"@pushrocks/smartpromise": "^2.0.5",
|
||||||
|
"@types/luxon": "^1.4.0",
|
||||||
|
"luxon": "^1.4.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
"@pushrocks/tapbundle": {
|
"@pushrocks/tapbundle": {
|
||||||
"version": "3.0.7",
|
"version": "3.0.7",
|
||||||
"resolved": "https://registry.npmjs.org/@pushrocks/tapbundle/-/tapbundle-3.0.7.tgz",
|
"resolved": "https://registry.npmjs.org/@pushrocks/tapbundle/-/tapbundle-3.0.7.tgz",
|
||||||
@ -260,6 +270,11 @@
|
|||||||
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz",
|
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.118.tgz",
|
||||||
"integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw=="
|
"integrity": "sha512-iiJbKLZbhSa6FYRip/9ZDX6HXhayXLDGY2Fqws9cOkEQ6XeKfaxB0sC541mowZJueYyMnVUmmG+al5/4fCDrgw=="
|
||||||
},
|
},
|
||||||
|
"@types/luxon": {
|
||||||
|
"version": "1.4.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-1.4.1.tgz",
|
||||||
|
"integrity": "sha512-mYv/gbkOJ40CDgR8st5sosfFNrJncdlkpdzQSNRdU86UQg3oWWmll4AO/7B8F5FlBC6YrIXqXDSnkoCBqo+uMA=="
|
||||||
|
},
|
||||||
"@types/minimatch": {
|
"@types/minimatch": {
|
||||||
"version": "3.0.3",
|
"version": "3.0.3",
|
||||||
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
|
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
|
||||||
@ -717,6 +732,11 @@
|
|||||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
|
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
|
||||||
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
|
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
|
||||||
},
|
},
|
||||||
|
"luxon": {
|
||||||
|
"version": "1.5.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.5.0.tgz",
|
||||||
|
"integrity": "sha512-9IfJ0a5jA3poJSQnamaoOdPG0ulYk1JUI5IYmEdUhUc8dto4t6TVTu+oIM7fpSYy01GwUAA8kGUZMVe7Qx9Y+Q=="
|
||||||
|
},
|
||||||
"make-error": {
|
"make-error": {
|
||||||
"version": "1.3.5",
|
"version": "1.3.5",
|
||||||
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",
|
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",
|
||||||
|
@ -27,6 +27,7 @@
|
|||||||
"@pushrocks/smartdelay": "^2.0.2",
|
"@pushrocks/smartdelay": "^2.0.2",
|
||||||
"@pushrocks/smartlog-interfaces": "^2.0.2",
|
"@pushrocks/smartlog-interfaces": "^2.0.2",
|
||||||
"@pushrocks/smartpromise": "^2.0.5",
|
"@pushrocks/smartpromise": "^2.0.5",
|
||||||
|
"@pushrocks/smarttime": "^3.0.2",
|
||||||
"@types/elasticsearch": "^5.0.28",
|
"@types/elasticsearch": "^5.0.28",
|
||||||
"elasticsearch": "^15.2.0"
|
"elasticsearch": "^15.2.0"
|
||||||
}
|
}
|
||||||
|
@ -8,6 +8,8 @@ let testElasticLog: elasticsearch.ElasticSearch<any>;
|
|||||||
|
|
||||||
tap.test('first test', async () => {
|
tap.test('first test', async () => {
|
||||||
testElasticLog = new elasticsearch.ElasticSearch({
|
testElasticLog = new elasticsearch.ElasticSearch({
|
||||||
|
indexPrefix: 'smartlog',
|
||||||
|
indexRetention: 7,
|
||||||
domain: process.env.ELK_DOMAIN,
|
domain: process.env.ELK_DOMAIN,
|
||||||
port: parseInt(process.env.ELK_PORT, 10),
|
port: parseInt(process.env.ELK_PORT, 10),
|
||||||
ssl: true,
|
ssl: true,
|
||||||
|
@ -14,32 +14,68 @@ export class ElasticIndex {
|
|||||||
|
|
||||||
public async ensureIndex(indexArg: string) {
|
public async ensureIndex(indexArg: string) {
|
||||||
const done = plugins.smartpromise.defer();
|
const done = plugins.smartpromise.defer();
|
||||||
if(this.stringmap.checkString(indexArg)) {
|
if (this.stringmap.checkString(indexArg)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.elasticSearchRef.client.cat.indices({
|
this.elasticSearchRef.client.cat.indices(
|
||||||
format: 'json',
|
{
|
||||||
bytes: 'm'
|
format: 'json',
|
||||||
}, async (err, response: any[]) => {
|
bytes: 'm'
|
||||||
// console.log(response);
|
},
|
||||||
const index = response.find(indexObject => {
|
async (err, response: any[]) => {
|
||||||
return indexObject.index === indexArg;
|
// lets delete indexes that violate the retention
|
||||||
});
|
const filteredIndices = response.filter(indexObjectArg => {
|
||||||
|
return indexObjectArg.index.startsWith('smartlog');
|
||||||
if(!index) {
|
|
||||||
const done2 = plugins.smartpromise.defer();
|
|
||||||
this.elasticSearchRef.client.indices.create({
|
|
||||||
waitForActiveShards: '2',
|
|
||||||
index: indexArg
|
|
||||||
}, (error, response) => {
|
|
||||||
// console.lof(response)
|
|
||||||
done2.resolve();
|
|
||||||
});
|
});
|
||||||
await done2.promise;
|
const filteredIndexNames = filteredIndices.map(indexObjectArg => {
|
||||||
|
return indexObjectArg.name;
|
||||||
|
});
|
||||||
|
const todayAsUnix: number = Date.now();
|
||||||
|
const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
|
||||||
|
this.elasticSearchRef.indexRetention
|
||||||
|
);
|
||||||
|
for (const indexName of filteredIndexNames) {
|
||||||
|
const regexResult = /^smartlog-([0-9]*)\.([0-9]*)\.([0-9]*)$/;
|
||||||
|
const dateAsUnix: number = new Date(
|
||||||
|
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
|
||||||
|
).getTime();
|
||||||
|
if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
|
||||||
|
const done2 = plugins.smartpromise.defer();
|
||||||
|
this.elasticSearchRef.client.indices.delete({
|
||||||
|
index: indexName
|
||||||
|
}, (err2, response2) => {
|
||||||
|
if(err2) {
|
||||||
|
console.log(err2);
|
||||||
|
}
|
||||||
|
done.resolve();
|
||||||
|
});
|
||||||
|
await done2.promise;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// console.log(response);
|
||||||
|
const index = response.find(indexObject => {
|
||||||
|
return indexObject.index === indexArg;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!index) {
|
||||||
|
const done2 = plugins.smartpromise.defer();
|
||||||
|
this.elasticSearchRef.client.indices.create(
|
||||||
|
{
|
||||||
|
waitForActiveShards: '2',
|
||||||
|
index: indexArg
|
||||||
|
},
|
||||||
|
(error, response) => {
|
||||||
|
// console.lof(response)
|
||||||
|
done2.resolve();
|
||||||
|
}
|
||||||
|
);
|
||||||
|
await done2.promise;
|
||||||
|
}
|
||||||
|
this.stringmap.addString(indexArg);
|
||||||
|
done.resolve();
|
||||||
}
|
}
|
||||||
this.stringmap.addString(indexArg);
|
);
|
||||||
done.resolve();
|
|
||||||
});
|
|
||||||
await done.promise;
|
await done.promise;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -11,7 +11,9 @@ export interface IStandardLogParams {
|
|||||||
severity: string;
|
severity: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IElasticLogConstructorOptions {
|
export interface IElasticSearchConstructorOptions {
|
||||||
|
indexPrefix: string;
|
||||||
|
indexRetention: number;
|
||||||
port: number;
|
port: number;
|
||||||
domain: string;
|
domain: string;
|
||||||
ssl: boolean;
|
ssl: boolean;
|
||||||
@ -24,22 +26,27 @@ export class ElasticSearch<T> {
|
|||||||
public elasticScheduler = new ElasticScheduler(this);
|
public elasticScheduler = new ElasticScheduler(this);
|
||||||
public elasticIndex: ElasticIndex = new ElasticIndex(this);
|
public elasticIndex: ElasticIndex = new ElasticIndex(this);
|
||||||
|
|
||||||
|
public indexPrefix: string;
|
||||||
|
public indexRetention: number;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* sets up an instance of Elastic log
|
* sets up an instance of Elastic log
|
||||||
* @param optionsArg
|
* @param optionsArg
|
||||||
*/
|
*/
|
||||||
constructor(optionsArg: IElasticLogConstructorOptions) {
|
constructor(optionsArg: IElasticSearchConstructorOptions) {
|
||||||
this.client = new ElasticClient({
|
this.client = new ElasticClient({
|
||||||
host: this.computeHostString(optionsArg),
|
host: this.computeHostString(optionsArg),
|
||||||
log: 'trace'
|
log: 'trace'
|
||||||
});
|
});
|
||||||
|
this.indexPrefix = optionsArg.indexPrefix;
|
||||||
|
this.indexRetention = optionsArg.indexRetention;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* computes the host string from the constructor options
|
* computes the host string from the constructor options
|
||||||
* @param optionsArg
|
* @param optionsArg
|
||||||
*/
|
*/
|
||||||
private computeHostString(optionsArg: IElasticLogConstructorOptions): string {
|
private computeHostString(optionsArg: IElasticSearchConstructorOptions): string {
|
||||||
let hostString = `${optionsArg.domain}:${optionsArg.port}`;
|
let hostString = `${optionsArg.domain}:${optionsArg.port}`;
|
||||||
if (optionsArg.user && optionsArg.pass) {
|
if (optionsArg.user && optionsArg.pass) {
|
||||||
hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
|
hostString = `${optionsArg.user}:${optionsArg.pass}@${hostString}`;
|
||||||
@ -54,7 +61,7 @@ export class ElasticSearch<T> {
|
|||||||
|
|
||||||
public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
|
public async log(logPackageArg: ILogPackage, scheduleOverwrite = false) {
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
const indexToUse = `smartlog-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
|
const indexToUse = `${this.indexPrefix}-${now.getFullYear()}.${('0' + (now.getMonth() + 1)).slice(-2)}.${(
|
||||||
'0' + now.getDate()
|
'0' + now.getDate()
|
||||||
).slice(-2)}`;
|
).slice(-2)}`;
|
||||||
|
|
||||||
|
@ -2,5 +2,6 @@ import * as elasticsearch from 'elasticsearch';
|
|||||||
import * as smartdelay from '@pushrocks/smartdelay';
|
import * as smartdelay from '@pushrocks/smartdelay';
|
||||||
import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
|
import * as smartlogInterfaces from '@pushrocks/smartlog-interfaces';
|
||||||
import * as smartpromise from '@pushrocks/smartpromise';
|
import * as smartpromise from '@pushrocks/smartpromise';
|
||||||
|
import * as smarttime from '@pushrocks/smarttime';
|
||||||
|
|
||||||
export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise };
|
export { elasticsearch, smartdelay, smartlogInterfaces, smartpromise, smarttime };
|
||||||
|
Loading…
Reference in New Issue
Block a user