Compare commits

...

43 Commits

Author SHA1 Message Date
dda83e35c0 3.1.1 2024-11-26 22:41:11 +01:00
8bf1af08d3 fix(core): Fix S3 cache manager methods for better path encoding consistency 2024-11-26 22:41:10 +01:00
69b2aa7fb8 3.1.0 2024-11-24 20:16:25 +01:00
75140e9d70 feat(core): Enhanced caching solution with optional configurations and improved documentation. 2024-11-24 20:16:25 +01:00
e45c662658 update description 2024-05-29 14:11:10 +02:00
d9b0ebb5ed update tsconfig 2024-04-14 13:38:03 +02:00
f5075abafb update npmextra.json: githost 2024-04-01 21:33:14 +02:00
e131b8c9ff update npmextra.json: githost 2024-04-01 19:57:25 +02:00
244d229fa3 update npmextra.json: githost 2024-03-30 21:46:21 +01:00
96080c5de0 3.0.8 2024-02-14 01:33:57 +01:00
a981c63f9b fix(core): update 2024-02-14 01:33:56 +01:00
cd5e2f03f4 3.0.7 2024-02-14 01:32:20 +01:00
eb5c1f40ee fix(core): update 2024-02-14 01:32:19 +01:00
ed1d66ee2a 3.0.6 2023-07-21 00:44:40 +02:00
45122024fc fix(core): update 2023-07-21 00:44:39 +02:00
fcaed1a3a0 3.0.5 2023-07-21 00:44:07 +02:00
0ce5eba798 fix(core): update 2023-07-21 00:44:06 +02:00
9d6111a77f 3.0.4 2023-07-20 22:54:08 +02:00
ac11045812 fix(core): update 2023-07-20 22:54:07 +02:00
2266ee0c73 switch to new org scheme 2023-07-11 00:00:42 +02:00
e4f0a60644 switch to new org scheme 2023-07-10 02:41:30 +02:00
946e1cb7d3 3.0.3 2023-01-09 13:01:09 +01:00
6e5d17a793 fix(core): update 2023-01-09 13:01:08 +01:00
3f7a3c2786 3.0.2 2022-06-09 19:23:56 +02:00
925921a9fe fix(core): update 2022-06-09 19:23:55 +02:00
61ccf38d85 3.0.1 2022-04-04 14:18:39 +02:00
061a79b0fd update 2022-04-04 14:18:36 +02:00
2f4fb82918 3.0.0 2022-04-02 17:40:38 +02:00
266da3b476 2.0.1 2022-03-22 23:15:08 +01:00
074767b294 fix(core): update 2022-03-22 23:15:08 +01:00
85abc4da3d 2.0.0 2022-03-22 22:45:13 +01:00
716d805723 BREAKING CHANGE(core): switch to esm 2022-03-22 22:45:12 +01:00
9feaa921bf 1.0.10 2021-05-10 14:26:32 +00:00
6b57e8b1f3 fix(caching): properly respect ttl for all cache levels 2021-05-10 14:26:32 +00:00
f7b6df5ff7 1.0.9 2021-04-23 18:50:21 +00:00
6a4df84f4f 1.0.8 2021-04-23 18:41:30 +00:00
17b59fd222 fix(core): update 2021-04-23 18:41:30 +00:00
d18f4b6928 1.0.7 2021-04-23 18:40:58 +00:00
e53ebc74f8 fix(core): update 2021-04-23 18:40:57 +00:00
be8227f451 1.0.6 2020-02-15 22:38:29 +00:00
d8f526ef2d fix(core): update 2020-02-15 22:38:28 +00:00
92b858e248 1.0.5 2020-02-14 18:10:10 +00:00
26511fc10e fix(core): update 2020-02-14 18:10:10 +00:00
27 changed files with 10746 additions and 2156 deletions

View File

@ -0,0 +1,66 @@
name: Default (not tags)
on:
push:
tags-ignore:
- '**'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
- name: Run npm prepare
run: npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build

View File

@ -0,0 +1,124 @@
name: Default (tags)
on:
push:
tags:
- '*'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Release
run: |
npmci node install stable
npmci npm publish
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Code quality
run: |
npmci command npm install -g typescript
npmci npm install
- name: Trigger
run: npmci trigger
- name: Build docs and upload artifacts
run: |
npmci node install stable
npmci npm install
pnpm install -g @git.zone/tsdoc
npmci command tsdoc
continue-on-error: true

4
.gitignore vendored
View File

@ -15,8 +15,6 @@ node_modules/
# builds
dist/
dist_web/
dist_serve/
dist_ts_web/
dist_*/
# custom

View File

@ -1,126 +0,0 @@
# gitzone ci_default
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
cache:
paths:
- .npmci_cache/
key: '$CI_BUILD_STAGE'
stages:
- security
- test
- release
- metadata
# ====================
# security stage
# ====================
mirror:
stage: security
script:
- npmci git mirror
tags:
- lossless
- docker
- notpriv
snyk:
image: registry.gitlab.com/hosttoday/ht-docker-node:snyk
stage: security
script:
- npmci npm prepare
- npmci command npm install --ignore-scripts
- npmci command snyk test
tags:
- lossless
- docker
- notpriv
# ====================
# test stage
# ====================
testStable:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- priv
testBuild:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- notpriv
release:
stage: release
script:
- npmci node install stable
- npmci npm publish
only:
- tags
tags:
- lossless
- docker
- notpriv
# ====================
# metadata stage
# ====================
codequality:
stage: metadata
allow_failure: true
script:
- npmci command npm install -g tslint typescript
- npmci npm prepare
- npmci npm install
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
tags:
- lossless
- docker
- priv
trigger:
stage: metadata
script:
- npmci trigger
only:
- tags
tags:
- lossless
- docker
- notpriv
pages:
stage: metadata
script:
- npmci node install lts
- npmci command npm install -g @gitzone/tsdoc
- npmci npm prepare
- npmci npm install
- npmci command tsdoc
tags:
- lossless
- docker
- notpriv
only:
- tags
artifacts:
expire_in: 1 week
paths:
- public
allow_failure: true

24
.vscode/launch.json vendored
View File

@ -2,28 +2,10 @@
"version": "0.2.0",
"configurations": [
{
"name": "current file",
"type": "node",
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"args": [
"${relativeFile}"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
},
{
"name": "test.ts",
"type": "node",
"request": "launch",
"args": [
"test/test.ts"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
"type": "node-terminal"
}
]
}

View File

@ -15,7 +15,7 @@
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm"]
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}

75
changelog.md Normal file
View File

@ -0,0 +1,75 @@
# Changelog
## 2024-11-26 - 3.1.1 - fix(core)
Fix S3 cache manager methods for better path encoding consistency
- Corrected path parameter usage in S3 manager methods to ensure encoding and consistency.
- Updated package.json Git dependencies for scoped packages.
## 2024-11-24 - 3.1.0 - feat(core)
Enhanced caching solution with optional configurations and improved documentation.
- Improved package description and keywords for better discoverability.
- Updated dependencies to latest versions.
- Added strict type checking in TypeScript configurations.
- Enhanced README with detailed usage instructions and examples.
## 2024-05-29 to 2024-02-14 - 3.0.8
Minor configuration updates and documentation changes.
- Updated project description.
- Modified tsconfig setup.
- Adjustments to npmextra.json regarding githost.
## 2024-02-14 - 3.0.7
Core enhancements and bug fixes.
- Fixed updates in the core module to improve stability.
## 2024-02-14 - 3.0.6
Core maintenance and updates.
- Implemented minor fixes in the core module.
## 2023-07-21 - 3.0.5
Addressed core module adjustments.
- Patched core module to rectify issues.
## 2023-07-20 - 3.0.4
Further improvements to core functionality.
- Additional fixes applied to the core component.
## 2023-07-11 to 2023-07-10 - 3.0.3
Organizational and structural changes.
- Transitioned to a new organizational scheme.
## 2023-01-09 - 3.0.2
Core module corrections.
- Resolved various issues within the core module.
## 2022-06-09 - 3.0.1
Continuous enhancements in core functionality.
- Continued bug fixes for core module efficiency.
## 2022-04-04 to 2022-04-02 - 3.0.0
Major release with significant updates.
## 2022-03-22 - 2.0.0
Significant breaking changes in core tech stack.
- BREAKING CHANGE: Transitioned core module to ECMAScript Modules (ESM).
## 2021-05-10 - 1.0.9
Caching improvements and optimization.
- Enhanced caching by properly respecting TTL across all cache levels.
## 2020-02-15 to 2020-02-05 - 1.0.6 to 1.0.1
Initial series of core module fixes and updates.
- Persistent efforts to stabilize and improve core functionalities.

View File

@ -2,17 +2,34 @@
"gitzone": {
"projectType": "npm",
"module": {
"githost": "gitlab.com",
"gitscope": "pushrocks",
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "levelcache",
"shortDescription": "a cache that uses memory/disk/s3 as backup",
"npmPackagename": "@pushrocks/levelcache",
"description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
"npmPackagename": "@push.rocks/levelcache",
"license": "MIT",
"projectDomain": "push.rocks"
"projectDomain": "push.rocks",
"keywords": [
"caching",
"data storage",
"memory cache",
"disk cache",
"S3 storage",
"data backup",
"multi-level caching",
"performance",
"persistent storage",
"data retrieval",
"Node.js",
"TypeScript"
]
}
},
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
},
"tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
}
}

1883
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,41 +1,73 @@
{
"name": "@pushrocks/levelcache",
"version": "1.0.4",
"name": "@push.rocks/levelcache",
"version": "3.1.1",
"private": false,
"description": "a cache that uses memory/disk/s3 as backup",
"main": "dist/index.js",
"typings": "dist/index.d.ts",
"description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/)",
"build": "(tsbuild)",
"format": "(gitzone format)"
"test": "(tstest test/ --web)",
"build": "(tsbuild --web --allowimplicitany)",
"buildDocs": "tsdoc",
"localPublish": "gitzone commit && pnpm run build && pnpm publish && pnpm publish --access public --registry=\"https://registry.npmjs.org\""
},
"devDependencies": {
"@gitzone/tsbuild": "^2.0.22",
"@gitzone/tstest": "^1.0.15",
"@pushrocks/tapbundle": "^3.0.7",
"@types/node": "^13.7.0",
"tslint": "^6.0.0",
"tslint-config-prettier": "^1.15.0"
"@git.zone/tsbuild": "^2.1.66",
"@git.zone/tsrun": "^1.2.44",
"@git.zone/tstest": "^1.0.77",
"@push.rocks/tapbundle": "^5.5.3",
"@types/node": "^22.9.3"
},
"dependencies": {
"@pushrocks/lik": "^3.0.16",
"@pushrocks/smartbucket": "^1.0.24",
"@pushrocks/smartcache": "^1.0.13",
"@pushrocks/smartfile": "^7.0.6",
"@pushrocks/smartstring": "^3.0.18"
"@push.rocks/lik": "^6.1.0",
"@push.rocks/smartbucket": "^3.3.3",
"@push.rocks/smartcache": "^1.0.13",
"@push.rocks/smartenv": "^5.0.12",
"@push.rocks/smartexit": "^1.0.23",
"@push.rocks/smartfile": "^11.0.21",
"@push.rocks/smartjson": "^5.0.20",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.4",
"@push.rocks/smartstring": "^4.0.15",
"@push.rocks/smartunique": "^3.0.9",
"@push.rocks/taskbuffer": "^3.1.7",
"@tsclass/tsclass": "^4.1.2"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_web/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
]
],
"browserslist": [
"last 1 chrome versions"
],
"keywords": [
"caching",
"data storage",
"memory cache",
"disk cache",
"S3 storage",
"data backup",
"multi-level caching",
"performance",
"persistent storage",
"data retrieval",
"Node.js",
"TypeScript"
],
"homepage": "https://code.foss.global/push.rocks/levelcache",
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/levelcache.git"
}
}

9689
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

1
readme.hints.md Normal file
View File

@ -0,0 +1 @@

185
readme.md
View File

@ -1,32 +1,173 @@
# @pushrocks/levelcache
a cache that uses memory/disk/s3 as backup
# @push.rocks/levelcache
A cache that utilizes memory, disk, and S3 for data storage and backup.
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/levelcache)
* [gitlab.com (source)](https://gitlab.com/pushrocks/levelcache)
* [github.com (source mirror)](https://github.com/pushrocks/levelcache)
* [docs (typedoc)](https://pushrocks.gitlab.io/levelcache/)
## Install
To install `@push.rocks/levelcache`, you can use npm or yarn:
## Status for master
[![pipeline status](https://gitlab.com/pushrocks/levelcache/badges/master/pipeline.svg)](https://gitlab.com/pushrocks/levelcache/commits/master)
[![coverage report](https://gitlab.com/pushrocks/levelcache/badges/master/coverage.svg)](https://gitlab.com/pushrocks/levelcache/commits/master)
[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/levelcache.svg)](https://www.npmjs.com/package/@pushrocks/levelcache)
[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/levelcache/badge.svg)](https://snyk.io/test/npm/@pushrocks/levelcache)
[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
```bash
npm install @push.rocks/levelcache --save
```
or
```bash
yarn add @push.rocks/levelcache
```
This installs `@push.rocks/levelcache` and adds it to your project's dependencies.
## Usage
Use TypeScript for best in class intellisense.
`@push.rocks/levelcache` provides a comprehensive solution for multi-level caching that takes advantage of memory, disk, and Amazon S3 storage, making it a versatile tool for data caching and backup. The package is built with TypeScript, enabling strict type checks and better development experience. Below, we'll explore how to effectively employ `@push.rocks/levelcache` in your projects, discussing its features and demonstrating its usage with code examples.
## Contribution
### 1. Overview
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
The `LevelCache` class handles all cache operations. It decides where to store data based on pre-configured thresholds corresponding to the data size and the total storage capacity allocated for each storage type (memory/disk/S3). This mechanism optimizes both speed and persistence, allowing for efficient data storage and retrieval.
For further information read the linked docs at the top of this readme.
### 2. Getting Started: Initialization
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
To use `@push.rocks/levelcache`, you'll need to import the main classes: `LevelCache` and `CacheEntry`. `LevelCache` is the primary class, while `CacheEntry` represents individual pieces of cached data.
[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)
```typescript
import { LevelCache, CacheEntry } from '@push.rocks/levelcache';
```
#### Initialization with Optional Configurations
To create a cache, instantiate the `LevelCache` class with desired configurations. You can specify the limits for memory and disk storage, setup S3 configurations if needed, and more.
```typescript
const myCache = new LevelCache({
cacheId: 'myUniqueCacheId', // Unique ID for cache delineation
maxMemoryStorageInMB: 10, // Maximum memory use in MB (default 0.5 MB)
maxDiskStorageInMB: 100, // Maximum disk space in MB (default 10 MB)
diskStoragePath: './myCache', // Path for storing disk cache; default is '.nogit'
s3Config: {
accessKeyId: 'yourAccessKeyId', // AWS S3 access key
secretAccessKey: 'yourSecretAccessKey', // Corresponding secret key
region: 'us-west-2' // AWS region, e.g., 'us-west-2'
},
s3BucketName: 'myBucketName', // Designated name for S3 bucket
immutableCache: false, // Whether stored cache entries should remain unaltered
persistentCache: true, // Should the cache persist upon restarts
});
```
### 3. Storing and Retrieving Data
`LevelCache` methods enable seamless data storage and retrieval, handling complexity under the hood.
#### Storing Data
Create a `CacheEntry` specifying the data content and time-to-live (`ttl`). Use `storeCacheEntryByKey` to add this entry to the cache.
```typescript
async function storeData() {
// Wait for cache to be ready before operations
await myCache.ready;
const entryContents = Buffer.from('Caching this data');
const myCacheEntry = new CacheEntry({
ttl: 7200000, // Time-to-live in milliseconds (2 hours)
contents: entryContents,
});
// Storing the cache entry associated with a specific key
await myCache.storeCacheEntryByKey('someDataKey', myCacheEntry);
}
```
#### Retrieving Data
Retrieve stored data using `retrieveCacheEntryByKey`. The retrieved `CacheEntry` will give access to the original data.
```typescript
async function retrieveData() {
const retrievedEntry = await myCache.retrieveCacheEntryByKey('someDataKey');
if (retrievedEntry) {
const data = retrievedEntry.contents.toString();
console.log(data); // Expected output: Caching this data
} else {
console.log('Data not found or expired.');
}
}
```
### 4. Key Management: Updating and Deleting
#### Deleting Cache Entries
Remove entries with `deleteCacheEntryByKey`, enabling clean cache management.
```typescript
async function deleteData() {
// Removes an entry using its unique key identifier
await myCache.deleteCacheEntryByKey('someDataKey');
}
```
### 5. Cache Cleaning
Often, managing storage limits or removing outdated data becomes essential. The library supports these scenarios.
#### Automated Cleaning
While cache entries will naturally expire with `ttl` values, you can force-remove outdated entries.
```typescript
// Clean outdated or expired entries
await myCache.cleanOutdated();
```
#### Full Cache Reset
Clear all entries, efficiently resetting your cache storage.
```typescript
// Flush entire cache content
await myCache.cleanAll();
```
### 6. Configuring and Managing Advanced Use Cases
The flexible nature of `@push.rocks/levelcache` grants additional customization suited for more advanced requirements.
#### Custom Route Management
For certain demands, you might want to specify distinct data handling policies or routing logic.
- Adjust S3 handling, size thresholds, or immutability options dynamically.
- Utilize internal API expansions defined within the library for fine-grained operations.
#### Handling Large Datasets
Tailor the cache levels (memory, disk, S3) to accommodate higher loads:
```typescript
const largeDatasetCache = new LevelCache({
cacheId: 'largeDatasetCache',
// Customize limits and behavior for particular patterns
maxMemoryStorageInMB: 1024, // 1 GB memory allocation
maxDiskStorageInMB: 2048, // 2 GB disk space allowance
maxS3StorageInMB: 10240, // 10 GB S3 backup buffering
});
```
With intelligent routing and management embedded, `LevelCache` ensures optimal trade-offs between speed and stability.
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

View File

@ -1,15 +1,43 @@
import { expect, tap } from '@pushrocks/tapbundle';
import * as levelcache from '../ts/index';
import { expect, tap } from '@push.rocks/tapbundle';
import * as levelcache from '../ts/index.js';
import { CacheEntry } from '../ts/index.js';
let testLevelCache: levelcache.LevelCache;
tap.test('should create a new levelcache instance', async () => {
testLevelCache = new levelcache.LevelCache();
expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
testLevelCache = new levelcache.LevelCache({
cacheId: 'myCache',
});
expect(testLevelCache).toBeInstanceOf(levelcache.LevelCache);
});
tap.test('should cache a value', async () => {
await testLevelCache.storeCacheEntryByKey(
'mykey',
new CacheEntry({
contents: Buffer.from('heythere'),
ttl: 10000,
typeInfo: 'string',
})
);
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result.contents.toString()).toEqual('heythere');
});
tap.test('should respect ttl', async (tools) => {
await testLevelCache.storeCacheEntryByKey(
'mykey',
new CacheEntry({
contents: Buffer.from('heythere'),
ttl: 1000,
typeInfo: 'string',
})
);
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result.contents.toString()).toEqual('heythere');
await tools!.delayFor(1100);
const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result2).toBeNull();
});
tap.start();

8
ts/00_commitinfo_data.ts Normal file
View File

@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/levelcache',
version: '3.1.1',
description: 'A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.'
}

View File

@ -1 +1,2 @@
export * from './levelcache.classes.levelcache';
export * from './levelcache.classes.levelcache.js';
export * from './levelcache.classes.cacheentry.js';

View File

@ -0,0 +1,38 @@
import { CacheEntry } from './levelcache.classes.cacheentry.js';
export abstract class AbstractCache {
public abstract ready: Promise<void>;
public abstract status: 'active' | 'inactive';
// Cache Entries
/**
* store a Blob
*/
public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
/**
* retrieve cache entry
*/
public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
/**
* checks for the presence of a key
* @param keyArg
*/
public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
/**
* delete a key
*/
public abstract deleteCacheEntryByKey(keyArg: string): Promise<void>;
/**
* clean the cache
*/
public abstract cleanOutdated(): Promise<void>;
/**
* cleans the complete cache
*/
public abstract cleanAll(): Promise<void>;
}

View File

@ -1,8 +1,71 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import * as paths from './levelcache.paths.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
/**
*
*/
export class CacheDiskManager {
export class CacheDiskManager extends AbstractCache {
private levelCacheRef: LevelCache;
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
public fsPath: string;
public maxCacheSizeInMb: number;
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.diskStoragePath) {
this.fsPath = plugins.path.join(
this.levelCacheRef.options.diskStoragePath,
this.levelCacheRef.options.cacheId
);
} else {
this.fsPath = plugins.path.join(paths.nogitDir, this.levelCacheRef.options.cacheId);
}
if (this.status === 'active') {
plugins.smartfile.fs.ensureDirSync(this.fsPath);
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const fileString = await plugins.smartfile.fs.toStringSync(
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
);
return CacheEntry.fromStorageJsonString(fileString);
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await plugins.smartfile.memory.toFs(
cacheEntryArg.foldToJson(),
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
public async deleteCacheEntryByKey(keyArg: string) {
await plugins.smartfile.fs.remove(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
public async cleanOutdated() {}
public async cleanAll() {
if (this.status === 'active') {
if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);
}
}
}
}

View File

@ -1,5 +1,50 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
export class CacheMemoryManager {
export class CacheMemoryManager extends AbstractCache {
private levelCacheRef: LevelCache;
private fastMap = new plugins.lik.FastMap<CacheEntry>();
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
this.status = 'active';
this.readyDeferred.resolve();
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
this.fastMap.addToMap(keyArg, cacheEntryArg, { force: true });
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
return this.fastMap.getByKey(keyArg);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
if (this.fastMap.getByKey(keyArg)) {
return true;
} else {
return false;
}
}
public async deleteCacheEntryByKey(keyArg: string) {
this.fastMap.removeFromMap(keyArg);
}
public async cleanOutdated() {}
public async cleanAll() {
this.fastMap.clean();
}
}

View File

@ -1,8 +1,87 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { LevelCache } from './levelcache.classes.levelcache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
/**
*
*/
export class CacheS3Manager {
export class CacheS3Manager extends AbstractCache {
private levelCacheRef: LevelCache;
private smartbucket: plugins.smartbucket.SmartBucket;
private s3CacheBucket: plugins.smartbucket.Bucket;
private s3CacheDir: plugins.smartbucket.Directory;
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.s3Config) {
this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
this.s3CacheBucket = await this.smartbucket.getBucketByName('');
this.s3CacheDir = await (
await this.s3CacheBucket.getBaseDirectory()
).getSubDirectoryByName(this.levelCacheRef.options.cacheId);
if (this.levelCacheRef.options.maxS3StorageInMB) {
console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
} else {
console.log(`s3 cache started without limit. Automatically applying timebox of 1 month`);
}
this.status = 'active';
} else {
this.status = 'inactive';
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const jsonFileString = (await this.s3CacheDir.fastGet({
path: encodeURIComponent(keyArg),
})).toString();
const cacheEntry = CacheEntry.fromStorageJsonString(jsonFileString);
return cacheEntry;
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await this.s3CacheDir.fastPut({
path: encodeURIComponent(keyArg),
contents: cacheEntryArg.toStorageJsonString()
});
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
const files = await this.s3CacheDir.listFiles();
for (const file of files) {
if (file.name === keyArg) {
return true;
}
}
return false;
}
public async deleteCacheEntryByKey(keyArg: string) {
if (this.status === 'active') {
await this.s3CacheDir.fastRemove({
path: encodeURIComponent(keyArg),
});
}
}
/**
* clean outdated
*/
public async cleanOutdated() {}
public async cleanAll() {
await this.s3CacheDir.delete({
mode: 'permanent',
});
}
}

View File

@ -1,5 +1,44 @@
import * as plugins from './levelcache.plugins';
export class CacheEntry {
import * as plugins from './levelcache.plugins.js';
export interface ICacheEntryConstructorOptions {
key?: string;
ttl: number;
typeInfo?: string;
contents: Buffer;
}
/**
* a CacheEntry
*/
export class CacheEntry
extends plugins.smartjson.Smartjson
implements ICacheEntryConstructorOptions
{
public static fromStorageJsonString(storageJsonString: string) {
return new CacheEntry(plugins.smartjson.parse(storageJsonString));
}
@plugins.smartjson.foldDec()
public key: string;
@plugins.smartjson.foldDec()
public ttl: number;
@plugins.smartjson.foldDec()
public createdAt: number;
@plugins.smartjson.foldDec()
public typeInfo: string;
@plugins.smartjson.foldDec()
contents: Buffer;
public toStorageJsonString(): string {
return this.foldToJson();
}
constructor(optionsArg: ICacheEntryConstructorOptions) {
super();
Object.assign(this, optionsArg);
}
}

View File

@ -0,0 +1,75 @@
import * as plugins from './levelcache.plugins.js';
import { LevelCache } from './levelcache.classes.levelcache.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
export class CacheRouter {
public levelCacheRef: LevelCache;
public cacheKeyMap = new plugins.lik.FastMap<AbstractCache>();
constructor(levelCacheRef: LevelCache) {
this.levelCacheRef = levelCacheRef;
}
/**
* gets the relevant cache to perform a store action on
*/
async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
let returnCache: AbstractCache;
const mbToBytesMultiplier = 1000 * 1000;
const maxMemoryBytes = this.levelCacheRef.options.maxMemoryStorageInMB * mbToBytesMultiplier;
const maxDiskBytes = this.levelCacheRef.options.maxDiskStorageInMB * mbToBytesMultiplier;
const maxS3Bytes = this.levelCacheRef.options.maxS3StorageInMB * mbToBytesMultiplier;
switch (true) {
case cacheEntry.contents.byteLength <= maxMemoryBytes &&
this.levelCacheRef.cacheMemoryManager.status === 'active':
returnCache = this.levelCacheRef.cacheMemoryManager;
break;
case this.levelCacheRef.cacheDiskManager.status === 'active' &&
cacheEntry.contents.byteLength > maxMemoryBytes &&
cacheEntry.contents.byteLength <= maxDiskBytes &&
this.levelCacheRef.cacheDiskManager.status === 'active':
returnCache = this.levelCacheRef.cacheDiskManager;
break;
case cacheEntry.contents.byteLength > maxDiskBytes &&
cacheEntry.contents.byteLength < maxS3Bytes &&
this.levelCacheRef.cacheS3Manager.status === 'active':
returnCache = this.levelCacheRef.cacheS3Manager;
break;
default:
returnCache = null;
}
this.cacheKeyMap.addToMap(keyArg, returnCache);
return returnCache;
}
/**
* gets the relevant cache to perform a retrieval action on
*/
async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
const done = plugins.smartpromise.defer<AbstractCache>();
const returnCache = this.cacheKeyMap.getByKey(keyArg);
if (!returnCache && this.levelCacheRef.options.persistentCache) {
const checkCache = (cacheArg: AbstractCache) => {
const resultPromise = cacheArg.checkKeyPresence(keyArg);
resultPromise.then((hasKeyArg) => {
if (hasKeyArg) {
done.resolve(cacheArg);
}
});
return resultPromise;
};
Promise.all([
checkCache(this.levelCacheRef.cacheMemoryManager),
checkCache(this.levelCacheRef.cacheDiskManager),
checkCache(this.levelCacheRef.cacheMemoryManager),
]).then(() => {
done.resolve(returnCache);
});
} else {
done.resolve(returnCache);
}
return done.promise;
}
}

View File

@ -1,42 +1,121 @@
import * as plugins from './levelcache.plugins';
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager';
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
import * as plugins from './levelcache.plugins.js';
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager.js';
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager.js';
import { CacheS3Manager } from './levelcache.classes.cache.s3manager.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
import { CacheRouter } from './levelcache.classes.cacherouter.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
export interface ILevelCacheConstructorOptions {
/**
* a unique id when having more than one cache
*/
cacheId: string;
maxMemoryStorageInMB?: number;
maxDiskStorageInMB?: number;
maxS3StorageInMB?: number;
diskStoragePath?: string;
s3Config?: plugins.tsclass.storage.IS3Descriptor;
s3BucketName?: string;
forceLevel?: 'memory' | 'disk' | 's3';
expirationInMs?: number;
immutableCache?: boolean;
persistentCache?: boolean;
}
/**
* a leveled cache for storing things for a short time
*/
export class LevelCache {
public cacheMap = new plugins.lik.Objectmap();
public cacheDiskManager = new CacheDiskManager();
public cacheMemoryManager = new CacheMemoryManager();
public cacheS3Manager = new CacheS3Manager();
export class LevelCache extends AbstractCache {
private readyDeferred = plugins.smartpromise.defer<void>();
// Blobs
/**
* store a Blob
*/
public async storeBlobByKey () {};
public ready = this.readyDeferred.promise;
/**
* retrieve a blob
*/
public async retrieveBlob () {};
public status: 'active' = 'active'; // artifact of AbstractCache
public cacheRouter = new CacheRouter(this);
public cacheDiskManager: CacheDiskManager;
public cacheMemoryManager: CacheMemoryManager;
public cacheS3Manager: CacheS3Manager;
// Cache Entries
/**
* store a Cache Entries
*/
public async storeCacheEntry() {};
public options: ILevelCacheConstructorOptions;
constructor(optionsArg: ILevelCacheConstructorOptions) {
super();
this.options = {
maxMemoryStorageInMB: 0.5,
maxDiskStorageInMB: 10,
maxS3StorageInMB: 50,
...optionsArg,
};
this.init();
}
public async init() {
this.cacheMemoryManager = new CacheMemoryManager(this);
this.cacheDiskManager = new CacheDiskManager(this);
this.cacheS3Manager = new CacheS3Manager(this);
await Promise.all([
this.cacheDiskManager.ready,
this.cacheMemoryManager.ready,
this.cacheS3Manager.ready,
]);
this.readyDeferred.resolve();
}
// store things
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
cacheEntryArg.key = keyArg;
const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
cacheEntryArg.createdAt = Date.now();
await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
}
// retrieve things
/**
* retrieve cache entry
*/
public async retrieveCacheEntry () {};
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
if (targetCache) {
const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
if (cacheEntry.createdAt + cacheEntry.ttl < Date.now()) {
await this.deleteCacheEntryByKey(keyArg).catch();
return null;
}
return cacheEntry;
} else {
return null;
}
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
return plugins.smartpromise.getFirstTrueOrFalse([
this.cacheMemoryManager.checkKeyPresence(keyArg),
this.cacheDiskManager.checkKeyPresence(keyArg),
this.cacheS3Manager.checkKeyPresence(keyArg),
]);
}
public async deleteCacheEntryByKey(keyArg) {
await Promise.all([
this.cacheMemoryManager.deleteCacheEntryByKey(keyArg),
this.cacheDiskManager.deleteCacheEntryByKey(keyArg),
this.cacheS3Manager.deleteCacheEntryByKey(keyArg),
]);
}
// cache maintenance
public async cleanOutdated() {}
/**
* cleans the cache
* cleans the complete cache
*/
public clean() {};
public async cleanAll(): Promise<void> {
await Promise.all([
this.cacheDiskManager.cleanAll(),
this.cacheDiskManager.cleanAll(),
this.cacheS3Manager.cleanAll(),
]);
}
}

View File

@ -0,0 +1,7 @@
import * as plugins from './levelcache.plugins.js';
export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);
export const nogitDir = plugins.path.join(packageDir, '.nogit/');

View File

@ -1,21 +1,36 @@
// node native scope
import * as path from 'path';
export {
path
}
export { path };
// @pushrocks scope
import * as lik from '@pushrocks/lik';
import * as smartbucket from '@pushrocks/smartbucket';
import * as smartcache from '@pushrocks/smartcache';
import * as smartfile from '@pushrocks/smartfile';
import * as smartstring from '@pushrocks/smartstring';
// @push.rocks scope
import * as lik from '@push.rocks/lik';
import * as smartbucket from '@push.rocks/smartbucket';
import * as smartcache from '@push.rocks/smartcache';
import * as smartfile from '@push.rocks/smartfile';
import * as smartjson from '@push.rocks/smartjson';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartstring from '@push.rocks/smartstring';
import * as smartunique from '@push.rocks/smartunique';
import * as taskbuffer from '@push.rocks/taskbuffer';
export {
lik,
smartbucket,
smartcache,
smartfile,
smartstring
smartjson,
smartpath,
smartpromise,
smartstring,
smartunique,
taskbuffer,
};
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export {
tsclass,
};

14
tsconfig.json Normal file
View File

@ -0,0 +1,14 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}

View File

@ -1,17 +0,0 @@
{
"extends": ["tslint:latest", "tslint-config-prettier"],
"rules": {
"semicolon": [true, "always"],
"no-console": false,
"ordered-imports": false,
"object-literal-sort-keys": false,
"member-ordering": {
"options":{
"order": [
"static-method"
]
}
}
},
"defaultSeverity": "warning"
}