diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 991cfac..f9c5034 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/elasticsearch', - version: '2.0.3', + version: '2.0.4', description: 'log to elasticsearch in a kibana compatible format' } diff --git a/ts/els.classes.elasticdoc.ts b/ts/els.classes.elasticdoc.ts index fb09520..ffa2fd1 100644 --- a/ts/els.classes.elasticdoc.ts +++ b/ts/els.classes.elasticdoc.ts @@ -71,26 +71,34 @@ export class ElasticDoc { async startPipingSession(options: { onlyNew?: boolean }) { this.sessionDocs.clear(); this.onlyNew = options.onlyNew; + if (this.onlyNew) { - // Retrieve and store the latest timestamp - const response = await this.client.search({ - index: this.index, - sort: '@timestamp:desc', - size: 1, - }); - const hit = response.body.hits.hits[0]; - this.latestTimestamp = hit?._source?.['@timestamp'] || null; - if (this.latestTimestamp) { - console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`); + // Check if index exists before attempting to search for the latest timestamp + const { body: indexExists } = await this.client.indices.exists({ index: this.index }); + + if (indexExists) { + const response = await this.client.search({ + index: this.index, + sort: '@timestamp:desc', + size: 1, + }); + const hit = response.body.hits.hits[0]; + this.latestTimestamp = hit?._source?.['@timestamp'] || null; + + if (this.latestTimestamp) { + console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`); + } else { + console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`); + } } else { - `Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.` + this.latestTimestamp = null; + console.log(`Index ${this.index} does not exist. Working in "onlyNew" mode, but will process all documents as the index is empty.`); } } } + async pipeDocument(optionsArg: { docId: string; timestamp?: string | number; doc: any }) { - // If 'onlyNew' is true, compare the document timestamp with the latest timestamp - await this.ensureIndexExists(optionsArg.doc); const documentBody = { @@ -98,6 +106,7 @@ export class ElasticDoc { ...(optionsArg.timestamp && { '@timestamp': optionsArg.timestamp }), }; + // If 'onlyNew' is true, compare the document timestamp with the latest timestamp if (this.onlyNew) { if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) { // Omit the document