apiclient.xyz/elasticsearch
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ci): Update CI workflows and build config; bump dependencies; code style and TS config fixes

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-29 09:48:31 +00:00
parent 36d7cb69a3
commit 0701207acd
20 changed files with 7858 additions and 5194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ts/00_commitinfo_data.ts
View File

@@ -1,8 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@apiclient.xyz/elasticsearch',
version: '2.0.16',
version: '2.0.17',
description: 'log to elasticsearch in a kibana compatible format'
}

84
ts/els.classes.elasticdoc.ts
View File

@@ -16,7 +16,7 @@ export interface ISnapshot {
export type SnapshotProcessor = (
iterator: AsyncIterable<any>,
prevSnapshot: ISnapshot | null
prevSnapshot: ISnapshot | null,
) => Promise<ISnapshot>;
export class ElasticDoc {
@@ -40,16 +40,16 @@ export class ElasticDoc {
private async ensureIndexExists(doc: any) {
if (!this.indexInitialized) {
const indexExists = await this.client.indices.exists({ index: this.index });
const indexExists = await this.client.indices.exists({
index: this.index,
});
if (!indexExists) {
const mappings = this.createMappingsFromDoc(doc);
await this.client.indices.create({
index: this.index,
body: {
// mappings,
settings: {
// You can define the settings according to your requirements here
},
// mappings,
settings: {
// You can define the settings according to your requirements here
},
});
}
@@ -64,7 +64,9 @@ export class ElasticDoc {
properties[key] = { type: 'date' };
continue;
}
properties[key] = { type: typeof doc[key] === 'number' ? 'float' : 'text' };
properties[key] = {
type: typeof doc[key] === 'number' ? 'float' : 'text',
};
}
return { properties };
}
@@ -85,18 +87,26 @@ export class ElasticDoc {
this.latestTimestamp = hit?._source?.['@timestamp'] || null;
if (this.latestTimestamp) {
console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
console.log(
`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`,
);
} else {
console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
console.log(
`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`,
);
}
} else if (this.onlyNew && !indexExists) {
console.log(`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`);
console.log(
`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`,
);
}
}
async pipeDocument(optionsArg: { docId: string; timestamp?: string | number; doc: any }) {
async pipeDocument(optionsArg: {
docId: string;
timestamp?: string | number;
doc: any;
}) {
await this.ensureIndexExists(optionsArg.doc);
const documentBody = {
@@ -106,7 +116,10 @@ export class ElasticDoc {
// If 'onlyNew' is true, compare the document timestamp with the latest timestamp
if (this.onlyNew) {
if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) {
if (
this.latestTimestamp &&
optionsArg.timestamp <= this.latestTimestamp
) {
this.fastForward = true;
} else {
this.fastForward = false;
@@ -140,7 +153,10 @@ export class ElasticDoc {
if (!response.hits.hits.length) {
break;
}
response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
response = await this.client.scroll({
scroll_id: response._scroll_id,
scroll: '1m',
});
}
for (const docId of allDocIds) {
@@ -169,20 +185,20 @@ export class ElasticDoc {
async takeSnapshot(processIterator: SnapshotProcessor) {
const snapshotIndex = `${this.index}_snapshots`;
const indexExists = await this.client.indices.exists({ index: snapshotIndex });
const indexExists = await this.client.indices.exists({
index: snapshotIndex,
});
if (!indexExists) {
await this.client.indices.create({
index: snapshotIndex,
body: {
mappings: {
properties: {
date: {
type: 'date',
},
aggregationData: {
type: 'object',
enabled: true,
},
mappings: {
properties: {
date: {
type: 'date',
},
aggregationData: {
type: 'object',
enabled: true,
},
},
},
@@ -191,14 +207,19 @@ export class ElasticDoc {
const documentIterator = this.getDocumentIterator();
const newSnapshot = await processIterator(documentIterator, await this.getLastSnapshot());
const newSnapshot = await processIterator(
documentIterator,
await this.getLastSnapshot(),
);
await this.storeSnapshot(newSnapshot);
}
private async getLastSnapshot(): Promise<ISnapshot | null> {
const snapshotIndex = `${this.index}_snapshots`;
const indexExists = await this.client.indices.exists({ index: snapshotIndex });
const indexExists = await this.client.indices.exists({
index: snapshotIndex,
});
if (!indexExists) {
return null;
@@ -236,7 +257,10 @@ export class ElasticDoc {
break;
}
response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
response = await this.client.scroll({
scroll_id: response._scroll_id,
scroll: '1m',
});
}
}

53
ts/els.classes.elasticindex.ts
View File

@@ -16,12 +16,14 @@ export class ElasticIndex {
return indexNameArg;
}
const responseArg = await this.elasticSearchRef.client.cat.indices({
format: 'json',
bytes: 'mb',
}).catch(err => {
console.log(err);
});
const responseArg = await this.elasticSearchRef.client.cat
.indices({
format: 'json',
bytes: 'mb',
})
.catch((err) => {
console.log(err);
});
if (!responseArg) {
throw new Error('Could not get valid response from elastic search');
@@ -57,19 +59,17 @@ export class ElasticIndex {
const response = await this.elasticSearchRef.client.indices.create({
wait_for_active_shards: 1,
index: indexNameArg,
body: {
mappings: {
properties: {
'@timestamp': {
type: 'date',
},
logPackageArg: {
properties: {
payload: {
type: 'object',
dynamic: true
}
}
mappings: {
properties: {
'@timestamp': {
type: 'date',
},
logPackageArg: {
properties: {
payload: {
type: 'object',
dynamic: true,
},
},
},
},
@@ -80,21 +80,24 @@ export class ElasticIndex {
public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
const todayAsUnix: number = Date.now();
const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
this.elasticSearchRef.indexRetention
this.elasticSearchRef.indexRetention,
);
for (const indexName of indicesArray) {
if (!indexName.startsWith(prefixArg)) continue;
const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`)
const indexRegex = new RegExp(
`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`,
);
const regexResult = indexRegex.exec(indexName);
const dateAsUnix: number = new Date(
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`,
).getTime();
if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
console.log(`found old index ${indexName}`);
const response = await this.elasticSearchRef.client.indices.delete(
{
const response = await this.elasticSearchRef.client.indices
.delete({
index: indexName,
}).catch(err => {
})
.catch((err) => {
console.log(err);
});
}

11
ts/els.classes.elasticscheduler.ts
View File

@@ -1,4 +1,7 @@
import { ElsSmartlogDestination, type IStandardLogParams } from './els.classes.smartlogdestination.js';
import {
ElsSmartlogDestination,
type IStandardLogParams,
} from './els.classes.smartlogdestination.js';
export class ElasticScheduler {
elasticSearchRef: ElsSmartlogDestination<any>;
@@ -16,14 +19,14 @@ export class ElasticScheduler {
this.addToStorage(objectArg);
this.setRetry();
}
public scheduleDoc(logObject: any) {
this.addToStorage(logObject);
}
private addToStorage(logObject: any) {
this.docsStorage.push(logObject);
// if buffer is full, send logs immediately
if (this.docsStorage.length >= this.maxBufferSize) {
this.flushLogsToElasticSearch();
@@ -33,7 +36,7 @@ export class ElasticScheduler {
private flushLogsToElasticSearch() {
const oldStorage = this.docsStorage;
this.docsStorage = [];
for (let logObject of oldStorage) {
this.elasticSearchRef.log(logObject, true);
}

25
ts/els.classes.fastpush.ts
View File

@@ -1,8 +1,8 @@
import { Client as ElasticClient } from '@elastic/elasticsearch';
interface FastPushOptions {
deleteOldData?: boolean; // Clear the index
deleteIndex?: boolean; // Delete the entire index
deleteOldData?: boolean; // Clear the index
deleteIndex?: boolean; // Delete the entire index
}
export class FastPush {
@@ -15,7 +15,11 @@ export class FastPush {
});
}
async pushToIndex(indexName: string, docArray: any[], options?: FastPushOptions) {
async pushToIndex(
indexName: string,
docArray: any[],
options?: FastPushOptions,
) {
if (docArray.length === 0) return;
const indexExists = await this.client.indices.exists({ index: indexName });
@@ -26,11 +30,9 @@ export class FastPush {
} else if (options?.deleteOldData) {
await this.client.deleteByQuery({
index: indexName,
body: {
query: {
match_all: {}
}
}
query: {
match_all: {},
},
});
}
}
@@ -39,11 +41,8 @@ export class FastPush {
// Create index with mappings (for simplicity, we use dynamic mapping)
await this.client.indices.create({
index: indexName,
body: {
mappings: {
dynamic: "true"
// ... other specific mappings
},
mappings: {
dynamic: 'true',
},
});
}

44
ts/els.classes.kvstore.ts
View File

@@ -27,23 +27,23 @@ export class ElasticKVStore {
private async setupIndex() {
try {
const indexExists = await this.client.indices.exists({ index: this.index });
const indexExists = await this.client.indices.exists({
index: this.index,
});
if (!indexExists) {
await this.client.indices.create({
await this.client.indices.create({
index: this.index,
body: {
mappings: {
properties: {
key: {
type: 'keyword'
},
value: {
type: 'text'
}
}
}
}
mappings: {
properties: {
key: {
type: 'keyword',
},
value: {
type: 'text',
},
},
},
});
}
this.readyDeferred.resolve();
@@ -59,8 +59,8 @@ export class ElasticKVStore {
id: key,
body: {
key,
value
}
value,
},
});
}
@@ -70,7 +70,7 @@ export class ElasticKVStore {
try {
const response = await this.client.get({
index: this.index,
id: key
id: key,
});
return response._source['value'];
} catch (error) {
@@ -87,7 +87,7 @@ export class ElasticKVStore {
try {
await this.client.delete({
index: this.index,
id: key
id: key,
});
} catch (error) {
if (error.meta && error.meta.statusCode !== 404) {
@@ -101,11 +101,9 @@ export class ElasticKVStore {
await this.client.deleteByQuery({
index: this.index,
body: {
query: {
match_all: {}
}
}
query: {
match_all: {},
},
});
}
}

22
ts/els.classes.smartlogdestination.ts
View File

@@ -1,5 +1,9 @@
import { Client as ElasticClient } from '@elastic/elasticsearch';
import type { ILogContext, ILogPackage, ILogDestination } from '@push.rocks/smartlog-interfaces';
import type {
ILogContext,
ILogPackage,
ILogDestination,
} from '@push.rocks/smartlog-interfaces';
import { ElasticScheduler } from './els.classes.elasticscheduler.js';
import { ElasticIndex } from './els.classes.elasticindex.js';
@@ -54,15 +58,13 @@ export class ElsSmartlogDestination<T> {
return;
}
await this.client.index(
{
index: indexToUse,
body: {
'@timestamp': new Date(logPackageArg.timestamp).toISOString(),
...logPackageArg,
},
}
);
await this.client.index({
index: indexToUse,
body: {
'@timestamp': new Date(logPackageArg.timestamp).toISOString(),
...logPackageArg,
},
});
}
get logDestination(): ILogDestination {

9
ts/els.plugins.ts
View File

@@ -5,4 +5,11 @@ import * as smartlogInterfaces from '@push.rocks/smartlog-interfaces';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smarttime from '@push.rocks/smarttime';
export { elasticsearch, lik, smartdelay, smartlogInterfaces, smartpromise, smarttime };
export {
elasticsearch,
lik,
smartdelay,
smartlogInterfaces,
smartpromise,
smarttime,
};