fix(ci): Update CI workflows and build config; bump dependencies; code style and TS config fixes
This commit is contained in:
@@ -6,8 +6,8 @@ on:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
@@ -6,8 +6,8 @@ on:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
@@ -119,6 +119,6 @@ jobs:
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @gitzone/tsdoc
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
||||
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -3,7 +3,6 @@
|
||||
# artifacts
|
||||
coverage/
|
||||
public/
|
||||
pages/
|
||||
|
||||
# installs
|
||||
node_modules/
|
||||
@@ -17,4 +16,8 @@ node_modules/
|
||||
dist/
|
||||
dist_*/
|
||||
|
||||
# custom
|
||||
# AI
|
||||
.claude/
|
||||
.serena/
|
||||
|
||||
#------# custom
|
||||
125
.gitlab-ci.yml
125
.gitlab-ci.yml
@@ -1,125 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
before_script:
|
||||
- pnpm install -g pnpm
|
||||
- pnpm install -g @shipzone/npmci
|
||||
- npmci npm prepare
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
auditProductionDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command pnpm audit --audit-level=high --prod
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
auditDevDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command pnpm audit --audit-level=high --dev
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm install
|
||||
- npmci npm build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
only:
|
||||
- tags
|
||||
script:
|
||||
- npmci command npm install -g typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run buildDocs
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
||||
100
changelog.md
Normal file
100
changelog.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-11-29 - 2.0.17 - fix(ci)
|
||||
Update CI workflows and build config; bump dependencies; code style and TS config fixes
|
||||
|
||||
- Gitea workflows updated: swapped CI image to code.foss.global, adjusted NPMCI_COMPUTED_REPOURL and replaced @shipzone/npmci with @ship.zone/npmci; tsdoc package path updated.
|
||||
- Removed legacy .gitlab-ci.yml (migrated CI to .gitea workflows).
|
||||
- Bumped dependencies and devDependencies (e.g. @elastic/elasticsearch -> ^9.2.0, @git.zone/* packages, @push.rocks/* packages) and added repository/bugs/homepage/pnpm/packageManager metadata to package.json.
|
||||
- Tests updated: import path change to @git.zone/tstest/tapbundle and test runner export changed to default export (export default tap.start()).
|
||||
- TypeScript config changes: module and moduleResolution set to NodeNext and added exclude for dist_*/**/*.d.ts.
|
||||
- Code cleanups and formatting: normalized object/argument formatting, trailing commas, safer ElasticClient call shapes (explicit option objects), and minor refactors across ElasticDoc, FastPush, KVStore, ElasticIndex, ElasticScheduler and smartlog destination.
|
||||
- Added .gitignore entries for local AI tool directories and added readme.hints.md and npmextra.json.
|
||||
|
||||
## 2023-08-30 - 2.0.2..2.0.16 - core
|
||||
Series of maintenance releases and small bugfixes on the 2.0.x line.
|
||||
|
||||
- Multiple "fix(core): update" commits across 2.0.2 → 2.0.16 addressing small bugs and stability/maintenance issues.
|
||||
- No single large feature added in these patch releases; recommended to consult individual release diffs if you need a precise change per patch.
|
||||
|
||||
## 2023-08-25 - 2.0.0 - core
|
||||
Major 2.0.0 release containing core updates and the transition from the 1.x line.
|
||||
|
||||
- Bumped major version to 2.0.0 with core updates.
|
||||
- This release follows a breaking-change update introduced on the 1.x line (see 1.0.56 below). Review breaking changes before upgrading.
|
||||
|
||||
## 2023-08-25 - 1.0.56 - core (BREAKING CHANGE)
|
||||
Breaking change introduced on the 1.x line.
|
||||
|
||||
- BREAKING CHANGE: core updated. Consumers should review the change and adapt integration code before upgrading from 1.0.55 → 1.0.56 (or migrating to 2.0.x).
|
||||
|
||||
## 2023-08-18 - 1.0.40..1.0.55 - maintenance
|
||||
Maintenance and fixes across many 1.0.x releases (mid 2023).
|
||||
|
||||
- Numerous "fix(core): update" commits across 1.0.40 → 1.0.55 addressing stability and minor bug fixes.
|
||||
- Includes smaller testing updates (e.g., fix(test): update in the 1.0.x series).
|
||||
|
||||
## 2023-07-05 - 1.0.32..1.0.44 - maintenance
|
||||
Maintenance sweep in the 1.0.x line (July 2023).
|
||||
|
||||
- Multiple small core fixes and updates across these patch releases.
|
||||
- No large feature additions; stability and incremental improvements only.
|
||||
|
||||
## 2019-11-02 - 1.0.26..1.0.30 - maintenance
|
||||
Patch-level fixes and cleanup in late 2019.
|
||||
|
||||
- Several "fix(core): update" releases to address minor issues and keep dependencies up to date.
|
||||
|
||||
## 2018-11-10 - 1.0.20 - core
|
||||
Cleanup related to indices.
|
||||
|
||||
- fix(clean up old indices): update — housekeeping and cleanup of old indices.
|
||||
|
||||
## 2018-11-03 - 1.0.13 - core
|
||||
Security/tooling update.
|
||||
|
||||
- fix(core): add snyk — added Snyk related changes (security/scan tooling integration).
|
||||
|
||||
## 2018-09-15 - 1.0.11 - core
|
||||
Dependency and compatibility updates.
|
||||
|
||||
- fix(core): update dependencies and bonsai.io compatibility — updated dependencies and ensured compatibility with bonsai.io.
|
||||
|
||||
## 2018-08-12 - 1.0.9 - test
|
||||
Testing improvements.
|
||||
|
||||
- fix(test): update — improvements/adjustments to test suite.
|
||||
|
||||
## 2018-03-03 - 1.0.7 - system
|
||||
System-level change.
|
||||
|
||||
- "system change" — internal/system modification (no public API feature).
|
||||
|
||||
## 2018-01-27 - 1.0.4 - quality/style
|
||||
Coverage and style updates.
|
||||
|
||||
- adjust coverageTreshold — adjusted test coverage threshold.
|
||||
- update style / update — code style and minor cleanup.
|
||||
|
||||
## 2018-01-27 - 1.0.3 - core (feat)
|
||||
Winston logging integration (added, later removed in a subsequent release).
|
||||
|
||||
- feat(core): implement winston support — initial addition of Winston logging support.
|
||||
|
||||
## 2018-01-27 - 1.0.6 - winston (fix)
|
||||
Removal of previously added logging integration.
|
||||
|
||||
- fix(winston): remove winston — removed Winston integration introduced earlier.
|
||||
|
||||
## 2018-01-26 - 1.0.2 - core (feat)
|
||||
Index generation improvement.
|
||||
|
||||
- feat(core): update index generation — improvements to index generation logic.
|
||||
|
||||
## 2018-01-24 - 1.0.1 - core (initial)
|
||||
Project initial commit and initial cleanup.
|
||||
|
||||
- feat(core): initial commit — project bootstrap.
|
||||
- fix(core): cleanup — initial cleanup and adjustments after the first commit.
|
||||
|
||||
Note: Versions that only contain bare version-tag commits (commit messages identical to the version string) have been summarized as ranges above. For detailed per-patch changes consult individual release diffs.
|
||||
35
package.json
35
package.json
@@ -14,20 +14,19 @@
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.1.70",
|
||||
"@git.zone/tsrun": "^1.2.46",
|
||||
"@git.zone/tstest": "^1.0.80",
|
||||
"@push.rocks/qenv": "^6.0.2",
|
||||
"@push.rocks/tapbundle": "^5.0.15",
|
||||
"@types/node": "^20.5.7"
|
||||
"@git.zone/tsbuild": "^3.1.0",
|
||||
"@git.zone/tsrun": "^2.0.0",
|
||||
"@git.zone/tstest": "^3.1.3",
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@types/node": "^24.10.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "8.9.0",
|
||||
"@push.rocks/lik": "^6.0.5",
|
||||
"@elastic/elasticsearch": "^9.2.0",
|
||||
"@push.rocks/lik": "^6.2.2",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartlog-interfaces": "^3.0.0",
|
||||
"@push.rocks/smartpromise": "^4.0.2",
|
||||
"@push.rocks/smarttime": "^4.0.5"
|
||||
"@push.rocks/smartlog-interfaces": "^3.0.2",
|
||||
"@push.rocks/smartpromise": "^4.2.3",
|
||||
"@push.rocks/smarttime": "^4.1.1"
|
||||
},
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
@@ -44,5 +43,17 @@
|
||||
"type": "module",
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
]
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://gitlab.com/mojoio/elasticsearch.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://gitlab.com/mojoio/elasticsearch/issues"
|
||||
},
|
||||
"homepage": "https://gitlab.com/mojoio/elasticsearch#readme",
|
||||
"pnpm": {
|
||||
"overrides": {}
|
||||
},
|
||||
"packageManager": "pnpm@10.18.1+sha512.77a884a165cbba2d8d1c19e3b4880eee6d2fcabd0d879121e282196b80042351d5eb3ca0935fa599da1dc51265cc68816ad2bddd2a2de5ea9fdf92adbec7cd34"
|
||||
}
|
||||
|
||||
12142
pnpm-lock.yaml
generated
12142
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
3
readme.hints.md
Normal file
3
readme.hints.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# Project Readme Hints
|
||||
|
||||
This is the initial readme hints file.
|
||||
337
readme.md
337
readme.md
@@ -1,48 +1,317 @@
|
||||
# @mojoio/elasticsearch
|
||||
log to elasticsearch in a kibana compatible format
|
||||
# @apiclient.xyz/elasticsearch
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@mojoio/elasticsearch)
|
||||
* [gitlab.com (source)](https://gitlab.com/mojoio/elasticsearch)
|
||||
* [github.com (source mirror)](https://github.com/mojoio/elasticsearch)
|
||||
* [docs (typedoc)](https://mojoio.gitlab.io/elasticsearch/)
|
||||
> 🔍 **Modern TypeScript client for Elasticsearch with built-in Kibana compatibility and advanced logging features**
|
||||
|
||||
## Status for master
|
||||
A powerful, type-safe wrapper around the official Elasticsearch client that provides intelligent log management, document handling, key-value storage, and fast data ingestion - all optimized for production use.
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
## Issue Reporting and Security
|
||||
|
||||
## Usage
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
Use TypeScript for best in class instellisense.
|
||||
## Features ✨
|
||||
|
||||
For further information read the linked docs at the top of this README.
|
||||
- **🎯 SmartLog Destination** - Full-featured logging destination compatible with Kibana, automatic index rotation, and retention management
|
||||
- **📦 ElasticDoc** - Advanced document management with piping sessions, snapshots, and automatic cleanup
|
||||
- **🚀 FastPush** - High-performance bulk document insertion with automatic index management
|
||||
- **💾 KVStore** - Simple key-value storage interface backed by Elasticsearch
|
||||
- **🔧 TypeScript First** - Complete type safety with full TypeScript support
|
||||
- **🌊 Data Streams** - Built-in support for Elasticsearch data streams
|
||||
- **⚡ Production Ready** - Designed for high-throughput production environments
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
> | By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
|
||||
## Installation 📦
|
||||
|
||||
[](https://push.rocks)
|
||||
```bash
|
||||
npm install @apiclient.xyz/elasticsearch
|
||||
# or
|
||||
pnpm install @apiclient.xyz/elasticsearch
|
||||
```
|
||||
|
||||
## Contribute
|
||||
## Quick Start 🚀
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. + Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: [Contribute monthly :)](https://lossless.link/contribute)
|
||||
### SmartLog Destination
|
||||
|
||||
## Contribution
|
||||
Perfect for application logging with automatic index rotation and Kibana compatibility:
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
```typescript
|
||||
import { ElsSmartlogDestination } from '@apiclient.xyz/elasticsearch';
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
const logger = new ElsSmartlogDestination({
|
||||
indexPrefix: 'app-logs',
|
||||
indexRetention: 7, // Keep logs for 7 days
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'your-password',
|
||||
},
|
||||
});
|
||||
|
||||
## Legal
|
||||
> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
// Log messages that automatically appear in Kibana
|
||||
await logger.log({
|
||||
timestamp: Date.now(),
|
||||
type: 'increment',
|
||||
level: 'info',
|
||||
context: {
|
||||
company: 'YourCompany',
|
||||
companyunit: 'api-service',
|
||||
containerName: 'web-server',
|
||||
environment: 'production',
|
||||
runtime: 'node',
|
||||
zone: 'us-east-1',
|
||||
},
|
||||
message: 'User authentication successful',
|
||||
correlation: null,
|
||||
});
|
||||
```
|
||||
|
||||
### ElasticDoc - Document Management
|
||||
|
||||
Handle documents with advanced features like piping sessions and snapshots:
|
||||
|
||||
```typescript
|
||||
import { ElasticDoc } from '@apiclient.xyz/elasticsearch';
|
||||
|
||||
const docManager = new ElasticDoc({
|
||||
index: 'products',
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'your-password',
|
||||
},
|
||||
});
|
||||
|
||||
// Start a piping session to manage document lifecycle
|
||||
await docManager.startPipingSession({});
|
||||
|
||||
// Add or update documents
|
||||
await docManager.pipeDocument({
|
||||
docId: 'product-001',
|
||||
timestamp: new Date().toISOString(),
|
||||
doc: {
|
||||
name: 'Premium Widget',
|
||||
price: 99.99,
|
||||
inStock: true
|
||||
},
|
||||
});
|
||||
|
||||
await docManager.pipeDocument({
|
||||
docId: 'product-002',
|
||||
timestamp: new Date().toISOString(),
|
||||
doc: {
|
||||
name: 'Deluxe Gadget',
|
||||
price: 149.99,
|
||||
inStock: false
|
||||
},
|
||||
});
|
||||
|
||||
// End session - automatically removes documents not in this session
|
||||
await docManager.endPipingSession();
|
||||
|
||||
// Take and store snapshots with custom aggregations
|
||||
await docManager.takeSnapshot(async (iterator, prevSnapshot) => {
|
||||
const aggregationData = [];
|
||||
for await (const doc of iterator) {
|
||||
aggregationData.push(doc);
|
||||
}
|
||||
return {
|
||||
date: new Date().toISOString(),
|
||||
aggregationData,
|
||||
};
|
||||
});
|
||||
```
|
||||
|
||||
### FastPush - Bulk Data Ingestion
|
||||
|
||||
Efficiently push large datasets with automatic index management:
|
||||
|
||||
```typescript
|
||||
import { FastPush } from '@apiclient.xyz/elasticsearch';
|
||||
|
||||
const fastPush = new FastPush({
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'your-password',
|
||||
},
|
||||
});
|
||||
|
||||
const documents = [
|
||||
{ id: 1, name: 'Document 1', data: 'Some data' },
|
||||
{ id: 2, name: 'Document 2', data: 'More data' },
|
||||
// ... thousands more documents
|
||||
];
|
||||
|
||||
// Push all documents with automatic batching
|
||||
await fastPush.pushDocuments('bulk-data', documents, {
|
||||
deleteOldData: true, // Clear old data before inserting
|
||||
});
|
||||
```
|
||||
|
||||
### KVStore - Key-Value Storage
|
||||
|
||||
Simple key-value storage backed by the power of Elasticsearch:
|
||||
|
||||
```typescript
|
||||
import { KVStore } from '@apiclient.xyz/elasticsearch';
|
||||
|
||||
const kvStore = new KVStore({
|
||||
index: 'app-config',
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'your-password',
|
||||
},
|
||||
});
|
||||
|
||||
// Set values
|
||||
await kvStore.set('api-key', 'sk-1234567890');
|
||||
await kvStore.set('feature-flags', JSON.stringify({ newUI: true }));
|
||||
|
||||
// Get values
|
||||
const apiKey = await kvStore.get('api-key');
|
||||
console.log(apiKey); // 'sk-1234567890'
|
||||
|
||||
// Clear all data
|
||||
await kvStore.clear();
|
||||
```
|
||||
|
||||
## Core Classes 🏗️
|
||||
|
||||
### ElsSmartlogDestination
|
||||
|
||||
The main logging destination class that provides:
|
||||
- Automatic index rotation based on date
|
||||
- Configurable retention policies
|
||||
- Kibana-compatible log format
|
||||
- Data stream support
|
||||
- Built-in scheduler for maintenance tasks
|
||||
|
||||
### ElasticDoc
|
||||
|
||||
Advanced document management with:
|
||||
- Piping sessions for tracking document lifecycles
|
||||
- Automatic cleanup of stale documents
|
||||
- Snapshot functionality with custom processors
|
||||
- Iterator-based document access
|
||||
- Fast-forward mode for incremental processing
|
||||
|
||||
### FastPush
|
||||
|
||||
High-performance bulk operations:
|
||||
- Automatic batching for optimal performance
|
||||
- Index management (create, delete, clear)
|
||||
- Dynamic mapping support
|
||||
- Efficient bulk API usage
|
||||
|
||||
### KVStore
|
||||
|
||||
Simple key-value interface:
|
||||
- Elasticsearch-backed storage
|
||||
- Async/await API
|
||||
- Automatic index initialization
|
||||
- Clear and get operations
|
||||
|
||||
## Advanced Usage 🎓
|
||||
|
||||
### Index Rotation and Retention
|
||||
|
||||
```typescript
|
||||
const logger = new ElsSmartlogDestination({
|
||||
indexPrefix: 'myapp',
|
||||
indexRetention: 30, // Keep 30 days of logs
|
||||
node: 'http://localhost:9200',
|
||||
});
|
||||
|
||||
// Indices are automatically created as: myapp-2025-01-22
|
||||
// Old indices are automatically deleted after 30 days
|
||||
```
|
||||
|
||||
### Document Iteration
|
||||
|
||||
```typescript
|
||||
// Iterate over all documents in an index
|
||||
const iterator = docManager.getDocumentIterator();
|
||||
for await (const doc of iterator) {
|
||||
console.log(doc);
|
||||
}
|
||||
|
||||
// Only process new documents since last run
|
||||
docManager.fastForward = true;
|
||||
await docManager.startPipingSession({ onlyNew: true });
|
||||
```
|
||||
|
||||
### Custom Snapshots
|
||||
|
||||
```typescript
|
||||
await docManager.takeSnapshot(async (iterator, prevSnapshot) => {
|
||||
let totalValue = 0;
|
||||
let count = 0;
|
||||
|
||||
for await (const doc of iterator) {
|
||||
totalValue += doc._source.price;
|
||||
count++;
|
||||
}
|
||||
|
||||
return {
|
||||
date: new Date().toISOString(),
|
||||
aggregationData: {
|
||||
totalValue,
|
||||
averagePrice: totalValue / count,
|
||||
count,
|
||||
previousSnapshot: prevSnapshot,
|
||||
},
|
||||
};
|
||||
});
|
||||
```
|
||||
|
||||
## API Compatibility 🔄
|
||||
|
||||
This module is built on top of `@elastic/elasticsearch` v9.x and is compatible with:
|
||||
- Elasticsearch 8.x and 9.x clusters
|
||||
- Kibana 8.x and 9.x for log visualization
|
||||
- OpenSearch (with some limitations)
|
||||
|
||||
## TypeScript Support 💙
|
||||
|
||||
Full TypeScript support with comprehensive type definitions:
|
||||
|
||||
```typescript
|
||||
import type {
|
||||
IElasticDocConstructorOptions,
|
||||
ISnapshot,
|
||||
SnapshotProcessor
|
||||
} from '@apiclient.xyz/elasticsearch';
|
||||
```
|
||||
|
||||
## Performance Considerations ⚡
|
||||
|
||||
- **Bulk Operations**: FastPush uses 1000-document batches by default
|
||||
- **Connection Pooling**: Reuses Elasticsearch client connections
|
||||
- **Index Management**: Automatic index creation and deletion
|
||||
- **Data Streams**: Built-in support for efficient log ingestion
|
||||
|
||||
## Best Practices 💡
|
||||
|
||||
1. **Always use authentication** in production environments
|
||||
2. **Set appropriate retention policies** to manage storage costs
|
||||
3. **Use piping sessions** to automatically clean up stale documents
|
||||
4. **Leverage snapshots** for point-in-time analytics
|
||||
5. **Configure index templates** for consistent mappings
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import { Qenv } from '@push.rocks/qenv';
|
||||
import * as elasticsearch from '../ts/index.js';
|
||||
|
||||
@@ -12,8 +12,8 @@ tap.test('first test', async () => {
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'YourPassword'
|
||||
}
|
||||
password: 'YourPassword',
|
||||
},
|
||||
});
|
||||
expect(testElasticLog).toBeInstanceOf(elasticsearch.ElsSmartlogDestination);
|
||||
});
|
||||
@@ -42,8 +42,8 @@ tap.test('should create an ElasticDoc instance', async () => {
|
||||
node: 'http://localhost:9200',
|
||||
auth: {
|
||||
username: 'elastic',
|
||||
password: 'YourPassword'
|
||||
}
|
||||
password: 'YourPassword',
|
||||
},
|
||||
});
|
||||
expect(testElasticDoc).toBeInstanceOf(elasticsearch.ElasticDoc);
|
||||
});
|
||||
@@ -53,17 +53,17 @@ tap.test('should add and update documents in a piping session', async () => {
|
||||
await testElasticDoc.pipeDocument({
|
||||
docId: '1',
|
||||
timestamp: new Date().toISOString(),
|
||||
doc: { name: 'doc1' }
|
||||
doc: { name: 'doc1' },
|
||||
});
|
||||
await testElasticDoc.pipeDocument({
|
||||
docId: '2',
|
||||
timestamp: new Date().toISOString(),
|
||||
doc: { name: 'doc2' }
|
||||
doc: { name: 'doc2' },
|
||||
});
|
||||
await testElasticDoc.pipeDocument({
|
||||
docId: '1',
|
||||
timestamp: new Date().toISOString(),
|
||||
doc: { name: 'updated doc1' }
|
||||
doc: { name: 'updated doc1' },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -86,4 +86,4 @@ tap.test('should take and store snapshot', async () => {
|
||||
});
|
||||
});
|
||||
|
||||
tap.start();
|
||||
export default tap.start();
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @pushrocks/commitinfo
|
||||
* autocreated commitinfo by @push.rocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@apiclient.xyz/elasticsearch',
|
||||
version: '2.0.16',
|
||||
version: '2.0.17',
|
||||
description: 'log to elasticsearch in a kibana compatible format'
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface ISnapshot {
|
||||
|
||||
export type SnapshotProcessor = (
|
||||
iterator: AsyncIterable<any>,
|
||||
prevSnapshot: ISnapshot | null
|
||||
prevSnapshot: ISnapshot | null,
|
||||
) => Promise<ISnapshot>;
|
||||
|
||||
export class ElasticDoc {
|
||||
@@ -40,16 +40,16 @@ export class ElasticDoc {
|
||||
|
||||
private async ensureIndexExists(doc: any) {
|
||||
if (!this.indexInitialized) {
|
||||
const indexExists = await this.client.indices.exists({ index: this.index });
|
||||
const indexExists = await this.client.indices.exists({
|
||||
index: this.index,
|
||||
});
|
||||
if (!indexExists) {
|
||||
const mappings = this.createMappingsFromDoc(doc);
|
||||
await this.client.indices.create({
|
||||
index: this.index,
|
||||
body: {
|
||||
// mappings,
|
||||
settings: {
|
||||
// You can define the settings according to your requirements here
|
||||
},
|
||||
// mappings,
|
||||
settings: {
|
||||
// You can define the settings according to your requirements here
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -64,7 +64,9 @@ export class ElasticDoc {
|
||||
properties[key] = { type: 'date' };
|
||||
continue;
|
||||
}
|
||||
properties[key] = { type: typeof doc[key] === 'number' ? 'float' : 'text' };
|
||||
properties[key] = {
|
||||
type: typeof doc[key] === 'number' ? 'float' : 'text',
|
||||
};
|
||||
}
|
||||
return { properties };
|
||||
}
|
||||
@@ -85,18 +87,26 @@ export class ElasticDoc {
|
||||
this.latestTimestamp = hit?._source?.['@timestamp'] || null;
|
||||
|
||||
if (this.latestTimestamp) {
|
||||
console.log(`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`);
|
||||
console.log(
|
||||
`Working in "onlyNew" mode. Hence we are omitting documents prior to ${this.latestTimestamp}`,
|
||||
);
|
||||
} else {
|
||||
console.log(`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`);
|
||||
console.log(
|
||||
`Working in "onlyNew" mode, but no documents found in index ${this.index}. Hence processing all documents now.`,
|
||||
);
|
||||
}
|
||||
} else if (this.onlyNew && !indexExists) {
|
||||
console.log(`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`);
|
||||
console.log(
|
||||
`Working in "onlyNew" mode, but index ${this.index} does not exist. Hence processing all documents now.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
async pipeDocument(optionsArg: { docId: string; timestamp?: string | number; doc: any }) {
|
||||
async pipeDocument(optionsArg: {
|
||||
docId: string;
|
||||
timestamp?: string | number;
|
||||
doc: any;
|
||||
}) {
|
||||
await this.ensureIndexExists(optionsArg.doc);
|
||||
|
||||
const documentBody = {
|
||||
@@ -106,7 +116,10 @@ export class ElasticDoc {
|
||||
|
||||
// If 'onlyNew' is true, compare the document timestamp with the latest timestamp
|
||||
if (this.onlyNew) {
|
||||
if (this.latestTimestamp && optionsArg.timestamp <= this.latestTimestamp) {
|
||||
if (
|
||||
this.latestTimestamp &&
|
||||
optionsArg.timestamp <= this.latestTimestamp
|
||||
) {
|
||||
this.fastForward = true;
|
||||
} else {
|
||||
this.fastForward = false;
|
||||
@@ -140,7 +153,10 @@ export class ElasticDoc {
|
||||
if (!response.hits.hits.length) {
|
||||
break;
|
||||
}
|
||||
response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
|
||||
response = await this.client.scroll({
|
||||
scroll_id: response._scroll_id,
|
||||
scroll: '1m',
|
||||
});
|
||||
}
|
||||
|
||||
for (const docId of allDocIds) {
|
||||
@@ -169,20 +185,20 @@ export class ElasticDoc {
|
||||
async takeSnapshot(processIterator: SnapshotProcessor) {
|
||||
const snapshotIndex = `${this.index}_snapshots`;
|
||||
|
||||
const indexExists = await this.client.indices.exists({ index: snapshotIndex });
|
||||
const indexExists = await this.client.indices.exists({
|
||||
index: snapshotIndex,
|
||||
});
|
||||
if (!indexExists) {
|
||||
await this.client.indices.create({
|
||||
index: snapshotIndex,
|
||||
body: {
|
||||
mappings: {
|
||||
properties: {
|
||||
date: {
|
||||
type: 'date',
|
||||
},
|
||||
aggregationData: {
|
||||
type: 'object',
|
||||
enabled: true,
|
||||
},
|
||||
mappings: {
|
||||
properties: {
|
||||
date: {
|
||||
type: 'date',
|
||||
},
|
||||
aggregationData: {
|
||||
type: 'object',
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -191,14 +207,19 @@ export class ElasticDoc {
|
||||
|
||||
const documentIterator = this.getDocumentIterator();
|
||||
|
||||
const newSnapshot = await processIterator(documentIterator, await this.getLastSnapshot());
|
||||
const newSnapshot = await processIterator(
|
||||
documentIterator,
|
||||
await this.getLastSnapshot(),
|
||||
);
|
||||
|
||||
await this.storeSnapshot(newSnapshot);
|
||||
}
|
||||
|
||||
private async getLastSnapshot(): Promise<ISnapshot | null> {
|
||||
const snapshotIndex = `${this.index}_snapshots`;
|
||||
const indexExists = await this.client.indices.exists({ index: snapshotIndex });
|
||||
const indexExists = await this.client.indices.exists({
|
||||
index: snapshotIndex,
|
||||
});
|
||||
|
||||
if (!indexExists) {
|
||||
return null;
|
||||
@@ -236,7 +257,10 @@ export class ElasticDoc {
|
||||
break;
|
||||
}
|
||||
|
||||
response = await this.client.scroll({ scroll_id: response._scroll_id, scroll: '1m' });
|
||||
response = await this.client.scroll({
|
||||
scroll_id: response._scroll_id,
|
||||
scroll: '1m',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,12 +16,14 @@ export class ElasticIndex {
|
||||
return indexNameArg;
|
||||
}
|
||||
|
||||
const responseArg = await this.elasticSearchRef.client.cat.indices({
|
||||
format: 'json',
|
||||
bytes: 'mb',
|
||||
}).catch(err => {
|
||||
console.log(err);
|
||||
});
|
||||
const responseArg = await this.elasticSearchRef.client.cat
|
||||
.indices({
|
||||
format: 'json',
|
||||
bytes: 'mb',
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log(err);
|
||||
});
|
||||
|
||||
if (!responseArg) {
|
||||
throw new Error('Could not get valid response from elastic search');
|
||||
@@ -57,19 +59,17 @@ export class ElasticIndex {
|
||||
const response = await this.elasticSearchRef.client.indices.create({
|
||||
wait_for_active_shards: 1,
|
||||
index: indexNameArg,
|
||||
body: {
|
||||
mappings: {
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
type: 'date',
|
||||
},
|
||||
logPackageArg: {
|
||||
properties: {
|
||||
payload: {
|
||||
type: 'object',
|
||||
dynamic: true
|
||||
}
|
||||
}
|
||||
mappings: {
|
||||
properties: {
|
||||
'@timestamp': {
|
||||
type: 'date',
|
||||
},
|
||||
logPackageArg: {
|
||||
properties: {
|
||||
payload: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -80,21 +80,24 @@ export class ElasticIndex {
|
||||
public async deleteOldIndices(prefixArg: string, indicesArray: string[]) {
|
||||
const todayAsUnix: number = Date.now();
|
||||
const rententionPeriodAsUnix: number = plugins.smarttime.units.days(
|
||||
this.elasticSearchRef.indexRetention
|
||||
this.elasticSearchRef.indexRetention,
|
||||
);
|
||||
for (const indexName of indicesArray) {
|
||||
if (!indexName.startsWith(prefixArg)) continue;
|
||||
const indexRegex = new RegExp(`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`)
|
||||
const indexRegex = new RegExp(
|
||||
`^${prefixArg}-([0-9]*)-([0-9]*)-([0-9]*)$`,
|
||||
);
|
||||
const regexResult = indexRegex.exec(indexName);
|
||||
const dateAsUnix: number = new Date(
|
||||
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`
|
||||
`${regexResult[1]}-${regexResult[2]}-${regexResult[3]}`,
|
||||
).getTime();
|
||||
if (todayAsUnix - rententionPeriodAsUnix > dateAsUnix) {
|
||||
console.log(`found old index ${indexName}`);
|
||||
const response = await this.elasticSearchRef.client.indices.delete(
|
||||
{
|
||||
const response = await this.elasticSearchRef.client.indices
|
||||
.delete({
|
||||
index: indexName,
|
||||
}).catch(err => {
|
||||
})
|
||||
.catch((err) => {
|
||||
console.log(err);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { ElsSmartlogDestination, type IStandardLogParams } from './els.classes.smartlogdestination.js';
|
||||
import {
|
||||
ElsSmartlogDestination,
|
||||
type IStandardLogParams,
|
||||
} from './els.classes.smartlogdestination.js';
|
||||
|
||||
export class ElasticScheduler {
|
||||
elasticSearchRef: ElsSmartlogDestination<any>;
|
||||
@@ -16,14 +19,14 @@ export class ElasticScheduler {
|
||||
this.addToStorage(objectArg);
|
||||
this.setRetry();
|
||||
}
|
||||
|
||||
|
||||
public scheduleDoc(logObject: any) {
|
||||
this.addToStorage(logObject);
|
||||
}
|
||||
|
||||
private addToStorage(logObject: any) {
|
||||
this.docsStorage.push(logObject);
|
||||
|
||||
|
||||
// if buffer is full, send logs immediately
|
||||
if (this.docsStorage.length >= this.maxBufferSize) {
|
||||
this.flushLogsToElasticSearch();
|
||||
@@ -33,7 +36,7 @@ export class ElasticScheduler {
|
||||
private flushLogsToElasticSearch() {
|
||||
const oldStorage = this.docsStorage;
|
||||
this.docsStorage = [];
|
||||
|
||||
|
||||
for (let logObject of oldStorage) {
|
||||
this.elasticSearchRef.log(logObject, true);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { Client as ElasticClient } from '@elastic/elasticsearch';
|
||||
|
||||
interface FastPushOptions {
|
||||
deleteOldData?: boolean; // Clear the index
|
||||
deleteIndex?: boolean; // Delete the entire index
|
||||
deleteOldData?: boolean; // Clear the index
|
||||
deleteIndex?: boolean; // Delete the entire index
|
||||
}
|
||||
|
||||
export class FastPush {
|
||||
@@ -15,7 +15,11 @@ export class FastPush {
|
||||
});
|
||||
}
|
||||
|
||||
async pushToIndex(indexName: string, docArray: any[], options?: FastPushOptions) {
|
||||
async pushToIndex(
|
||||
indexName: string,
|
||||
docArray: any[],
|
||||
options?: FastPushOptions,
|
||||
) {
|
||||
if (docArray.length === 0) return;
|
||||
|
||||
const indexExists = await this.client.indices.exists({ index: indexName });
|
||||
@@ -26,11 +30,9 @@ export class FastPush {
|
||||
} else if (options?.deleteOldData) {
|
||||
await this.client.deleteByQuery({
|
||||
index: indexName,
|
||||
body: {
|
||||
query: {
|
||||
match_all: {}
|
||||
}
|
||||
}
|
||||
query: {
|
||||
match_all: {},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -39,11 +41,8 @@ export class FastPush {
|
||||
// Create index with mappings (for simplicity, we use dynamic mapping)
|
||||
await this.client.indices.create({
|
||||
index: indexName,
|
||||
body: {
|
||||
mappings: {
|
||||
dynamic: "true"
|
||||
// ... other specific mappings
|
||||
},
|
||||
mappings: {
|
||||
dynamic: 'true',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -27,23 +27,23 @@ export class ElasticKVStore {
|
||||
|
||||
private async setupIndex() {
|
||||
try {
|
||||
const indexExists = await this.client.indices.exists({ index: this.index });
|
||||
const indexExists = await this.client.indices.exists({
|
||||
index: this.index,
|
||||
});
|
||||
|
||||
if (!indexExists) {
|
||||
await this.client.indices.create({
|
||||
await this.client.indices.create({
|
||||
index: this.index,
|
||||
body: {
|
||||
mappings: {
|
||||
properties: {
|
||||
key: {
|
||||
type: 'keyword'
|
||||
},
|
||||
value: {
|
||||
type: 'text'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mappings: {
|
||||
properties: {
|
||||
key: {
|
||||
type: 'keyword',
|
||||
},
|
||||
value: {
|
||||
type: 'text',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
this.readyDeferred.resolve();
|
||||
@@ -59,8 +59,8 @@ export class ElasticKVStore {
|
||||
id: key,
|
||||
body: {
|
||||
key,
|
||||
value
|
||||
}
|
||||
value,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ export class ElasticKVStore {
|
||||
try {
|
||||
const response = await this.client.get({
|
||||
index: this.index,
|
||||
id: key
|
||||
id: key,
|
||||
});
|
||||
return response._source['value'];
|
||||
} catch (error) {
|
||||
@@ -87,7 +87,7 @@ export class ElasticKVStore {
|
||||
try {
|
||||
await this.client.delete({
|
||||
index: this.index,
|
||||
id: key
|
||||
id: key,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.meta && error.meta.statusCode !== 404) {
|
||||
@@ -101,11 +101,9 @@ export class ElasticKVStore {
|
||||
|
||||
await this.client.deleteByQuery({
|
||||
index: this.index,
|
||||
body: {
|
||||
query: {
|
||||
match_all: {}
|
||||
}
|
||||
}
|
||||
query: {
|
||||
match_all: {},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { Client as ElasticClient } from '@elastic/elasticsearch';
|
||||
import type { ILogContext, ILogPackage, ILogDestination } from '@push.rocks/smartlog-interfaces';
|
||||
import type {
|
||||
ILogContext,
|
||||
ILogPackage,
|
||||
ILogDestination,
|
||||
} from '@push.rocks/smartlog-interfaces';
|
||||
import { ElasticScheduler } from './els.classes.elasticscheduler.js';
|
||||
import { ElasticIndex } from './els.classes.elasticindex.js';
|
||||
|
||||
@@ -54,15 +58,13 @@ export class ElsSmartlogDestination<T> {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.client.index(
|
||||
{
|
||||
index: indexToUse,
|
||||
body: {
|
||||
'@timestamp': new Date(logPackageArg.timestamp).toISOString(),
|
||||
...logPackageArg,
|
||||
},
|
||||
}
|
||||
);
|
||||
await this.client.index({
|
||||
index: indexToUse,
|
||||
body: {
|
||||
'@timestamp': new Date(logPackageArg.timestamp).toISOString(),
|
||||
...logPackageArg,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
get logDestination(): ILogDestination {
|
||||
|
||||
@@ -5,4 +5,11 @@ import * as smartlogInterfaces from '@push.rocks/smartlog-interfaces';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smarttime from '@push.rocks/smarttime';
|
||||
|
||||
export { elasticsearch, lik, smartdelay, smartlogInterfaces, smartpromise, smarttime };
|
||||
export {
|
||||
elasticsearch,
|
||||
lik,
|
||||
smartdelay,
|
||||
smartlogInterfaces,
|
||||
smartpromise,
|
||||
smarttime,
|
||||
};
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"moduleResolution": "nodenext",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
}
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user