Compare commits
43 Commits
Author | SHA1 | Date | |
---|---|---|---|
dda83e35c0 | |||
8bf1af08d3 | |||
69b2aa7fb8 | |||
75140e9d70 | |||
e45c662658 | |||
d9b0ebb5ed | |||
f5075abafb | |||
e131b8c9ff | |||
244d229fa3 | |||
96080c5de0 | |||
a981c63f9b | |||
cd5e2f03f4 | |||
eb5c1f40ee | |||
ed1d66ee2a | |||
45122024fc | |||
fcaed1a3a0 | |||
0ce5eba798 | |||
9d6111a77f | |||
ac11045812 | |||
2266ee0c73 | |||
e4f0a60644 | |||
946e1cb7d3 | |||
6e5d17a793 | |||
3f7a3c2786 | |||
925921a9fe | |||
61ccf38d85 | |||
061a79b0fd | |||
2f4fb82918 | |||
266da3b476 | |||
074767b294 | |||
85abc4da3d | |||
716d805723 | |||
9feaa921bf | |||
6b57e8b1f3 | |||
f7b6df5ff7 | |||
6a4df84f4f | |||
17b59fd222 | |||
d18f4b6928 | |||
e53ebc74f8 | |||
be8227f451 | |||
d8f526ef2d | |||
92b858e248 | |||
26511fc10e |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
4
.gitignore
vendored
4
.gitignore
vendored
@ -15,8 +15,6 @@ node_modules/
|
||||
|
||||
# builds
|
||||
dist/
|
||||
dist_web/
|
||||
dist_serve/
|
||||
dist_ts_web/
|
||||
dist_*/
|
||||
|
||||
# custom
|
126
.gitlab-ci.yml
126
.gitlab-ci.yml
@ -1,126 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci git mirror
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
snyk:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:snyk
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command snyk test
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
script:
|
||||
- npmci command npm install -g tslint typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci command npm install -g @gitzone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
24
.vscode/launch.json
vendored
24
.vscode/launch.json
vendored
@ -2,28 +2,10 @@
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "current file",
|
||||
"type": "node",
|
||||
"command": "npm test",
|
||||
"name": "Run npm test",
|
||||
"request": "launch",
|
||||
"args": [
|
||||
"${relativeFile}"
|
||||
],
|
||||
"runtimeArgs": ["-r", "@gitzone/tsrun"],
|
||||
"cwd": "${workspaceRoot}",
|
||||
"protocol": "inspector",
|
||||
"internalConsoleOptions": "openOnSessionStart"
|
||||
},
|
||||
{
|
||||
"name": "test.ts",
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"args": [
|
||||
"test/test.ts"
|
||||
],
|
||||
"runtimeArgs": ["-r", "@gitzone/tsrun"],
|
||||
"cwd": "${workspaceRoot}",
|
||||
"protocol": "inspector",
|
||||
"internalConsoleOptions": "openOnSessionStart"
|
||||
"type": "node-terminal"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@ -15,7 +15,7 @@
|
||||
"properties": {
|
||||
"projectType": {
|
||||
"type": "string",
|
||||
"enum": ["website", "element", "service", "npm"]
|
||||
"enum": ["website", "element", "service", "npm", "wcc"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
75
changelog.md
Normal file
75
changelog.md
Normal file
@ -0,0 +1,75 @@
|
||||
# Changelog
|
||||
|
||||
## 2024-11-26 - 3.1.1 - fix(core)
|
||||
Fix S3 cache manager methods for better path encoding consistency
|
||||
|
||||
- Corrected path parameter usage in S3 manager methods to ensure encoding and consistency.
|
||||
- Updated package.json Git dependencies for scoped packages.
|
||||
|
||||
## 2024-11-24 - 3.1.0 - feat(core)
|
||||
Enhanced caching solution with optional configurations and improved documentation.
|
||||
|
||||
- Improved package description and keywords for better discoverability.
|
||||
- Updated dependencies to latest versions.
|
||||
- Added strict type checking in TypeScript configurations.
|
||||
- Enhanced README with detailed usage instructions and examples.
|
||||
|
||||
## 2024-05-29 to 2024-02-14 - 3.0.8
|
||||
Minor configuration updates and documentation changes.
|
||||
|
||||
- Updated project description.
|
||||
- Modified tsconfig setup.
|
||||
- Adjustments to npmextra.json regarding githost.
|
||||
|
||||
## 2024-02-14 - 3.0.7
|
||||
Core enhancements and bug fixes.
|
||||
|
||||
- Fixed updates in the core module to improve stability.
|
||||
|
||||
## 2024-02-14 - 3.0.6
|
||||
Core maintenance and updates.
|
||||
|
||||
- Implemented minor fixes in the core module.
|
||||
|
||||
## 2023-07-21 - 3.0.5
|
||||
Addressed core module adjustments.
|
||||
|
||||
- Patched core module to rectify issues.
|
||||
|
||||
## 2023-07-20 - 3.0.4
|
||||
Further improvements to core functionality.
|
||||
|
||||
- Additional fixes applied to the core component.
|
||||
|
||||
## 2023-07-11 to 2023-07-10 - 3.0.3
|
||||
Organizational and structural changes.
|
||||
|
||||
- Transitioned to a new organizational scheme.
|
||||
|
||||
## 2023-01-09 - 3.0.2
|
||||
Core module corrections.
|
||||
|
||||
- Resolved various issues within the core module.
|
||||
|
||||
## 2022-06-09 - 3.0.1
|
||||
Continuous enhancements in core functionality.
|
||||
|
||||
- Continued bug fixes for core module efficiency.
|
||||
|
||||
## 2022-04-04 to 2022-04-02 - 3.0.0
|
||||
Major release with significant updates.
|
||||
|
||||
## 2022-03-22 - 2.0.0
|
||||
Significant breaking changes in core tech stack.
|
||||
|
||||
- BREAKING CHANGE: Transitioned core module to ECMAScript Modules (ESM).
|
||||
|
||||
## 2021-05-10 - 1.0.9
|
||||
Caching improvements and optimization.
|
||||
|
||||
- Enhanced caching by properly respecting TTL across all cache levels.
|
||||
|
||||
## 2020-02-15 to 2020-02-05 - 1.0.6 to 1.0.1
|
||||
Initial series of core module fixes and updates.
|
||||
|
||||
- Persistent efforts to stabilize and improve core functionalities.
|
@ -2,17 +2,34 @@
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "pushrocks",
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "levelcache",
|
||||
"shortDescription": "a cache that uses memory/disk/s3 as backup",
|
||||
"npmPackagename": "@pushrocks/levelcache",
|
||||
"description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
|
||||
"npmPackagename": "@push.rocks/levelcache",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
"projectDomain": "push.rocks",
|
||||
"keywords": [
|
||||
"caching",
|
||||
"data storage",
|
||||
"memory cache",
|
||||
"disk cache",
|
||||
"S3 storage",
|
||||
"data backup",
|
||||
"multi-level caching",
|
||||
"performance",
|
||||
"persistent storage",
|
||||
"data retrieval",
|
||||
"Node.js",
|
||||
"TypeScript"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
}
|
||||
}
|
1883
package-lock.json
generated
1883
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
74
package.json
74
package.json
@ -1,41 +1,73 @@
|
||||
{
|
||||
"name": "@pushrocks/levelcache",
|
||||
"version": "1.0.4",
|
||||
"name": "@push.rocks/levelcache",
|
||||
"version": "3.1.1",
|
||||
"private": false,
|
||||
"description": "a cache that uses memory/disk/s3 as backup",
|
||||
"main": "dist/index.js",
|
||||
"typings": "dist/index.d.ts",
|
||||
"description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/)",
|
||||
"build": "(tsbuild)",
|
||||
"format": "(gitzone format)"
|
||||
"test": "(tstest test/ --web)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"buildDocs": "tsdoc",
|
||||
"localPublish": "gitzone commit && pnpm run build && pnpm publish && pnpm publish --access public --registry=\"https://registry.npmjs.org\""
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.0.22",
|
||||
"@gitzone/tstest": "^1.0.15",
|
||||
"@pushrocks/tapbundle": "^3.0.7",
|
||||
"@types/node": "^13.7.0",
|
||||
"tslint": "^6.0.0",
|
||||
"tslint-config-prettier": "^1.15.0"
|
||||
"@git.zone/tsbuild": "^2.1.66",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^1.0.77",
|
||||
"@push.rocks/tapbundle": "^5.5.3",
|
||||
"@types/node": "^22.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pushrocks/lik": "^3.0.16",
|
||||
"@pushrocks/smartbucket": "^1.0.24",
|
||||
"@pushrocks/smartcache": "^1.0.13",
|
||||
"@pushrocks/smartfile": "^7.0.6",
|
||||
"@pushrocks/smartstring": "^3.0.18"
|
||||
"@push.rocks/lik": "^6.1.0",
|
||||
"@push.rocks/smartbucket": "^3.3.3",
|
||||
"@push.rocks/smartcache": "^1.0.13",
|
||||
"@push.rocks/smartenv": "^5.0.12",
|
||||
"@push.rocks/smartexit": "^1.0.23",
|
||||
"@push.rocks/smartfile": "^11.0.21",
|
||||
"@push.rocks/smartjson": "^5.0.20",
|
||||
"@push.rocks/smartpath": "^5.0.18",
|
||||
"@push.rocks/smartpromise": "^4.0.4",
|
||||
"@push.rocks/smartstring": "^4.0.15",
|
||||
"@push.rocks/smartunique": "^3.0.9",
|
||||
"@push.rocks/taskbuffer": "^3.1.7",
|
||||
"@tsclass/tsclass": "^4.1.2"
|
||||
},
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
"ts_web/**/*",
|
||||
"dist/**/*",
|
||||
"dist_web/**/*",
|
||||
"dist_*/**/*",
|
||||
"dist_ts/**/*",
|
||||
"dist_ts_web/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
]
|
||||
],
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
],
|
||||
"keywords": [
|
||||
"caching",
|
||||
"data storage",
|
||||
"memory cache",
|
||||
"disk cache",
|
||||
"S3 storage",
|
||||
"data backup",
|
||||
"multi-level caching",
|
||||
"performance",
|
||||
"persistent storage",
|
||||
"data retrieval",
|
||||
"Node.js",
|
||||
"TypeScript"
|
||||
],
|
||||
"homepage": "https://code.foss.global/push.rocks/levelcache",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://code.foss.global/push.rocks/levelcache.git"
|
||||
}
|
||||
}
|
||||
|
9689
pnpm-lock.yaml
generated
Normal file
9689
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
1
readme.hints.md
Normal file
1
readme.hints.md
Normal file
@ -0,0 +1 @@
|
||||
|
185
readme.md
185
readme.md
@ -1,32 +1,173 @@
|
||||
# @pushrocks/levelcache
|
||||
a cache that uses memory/disk/s3 as backup
|
||||
# @push.rocks/levelcache
|
||||
A cache that utilizes memory, disk, and S3 for data storage and backup.
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/levelcache)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/levelcache)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/levelcache)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/levelcache/)
|
||||
## Install
|
||||
To install `@push.rocks/levelcache`, you can use npm or yarn:
|
||||
|
||||
## Status for master
|
||||
[](https://gitlab.com/pushrocks/levelcache/commits/master)
|
||||
[](https://gitlab.com/pushrocks/levelcache/commits/master)
|
||||
[](https://www.npmjs.com/package/@pushrocks/levelcache)
|
||||
[](https://snyk.io/test/npm/@pushrocks/levelcache)
|
||||
[](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
[](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
[](https://prettier.io/)
|
||||
```bash
|
||||
npm install @push.rocks/levelcache --save
|
||||
```
|
||||
or
|
||||
```bash
|
||||
yarn add @push.rocks/levelcache
|
||||
```
|
||||
|
||||
This installs `@push.rocks/levelcache` and adds it to your project's dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
Use TypeScript for best in class intellisense.
|
||||
`@push.rocks/levelcache` provides a comprehensive solution for multi-level caching that takes advantage of memory, disk, and Amazon S3 storage, making it a versatile tool for data caching and backup. The package is built with TypeScript, enabling strict type checks and better development experience. Below, we'll explore how to effectively employ `@push.rocks/levelcache` in your projects, discussing its features and demonstrating its usage with code examples.
|
||||
|
||||
## Contribution
|
||||
### 1. Overview
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
The `LevelCache` class handles all cache operations. It decides where to store data based on pre-configured thresholds corresponding to the data size and the total storage capacity allocated for each storage type (memory/disk/S3). This mechanism optimizes both speed and persistence, allowing for efficient data storage and retrieval.
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
### 2. Getting Started: Initialization
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
To use `@push.rocks/levelcache`, you'll need to import the main classes: `LevelCache` and `CacheEntry`. `LevelCache` is the primary class, while `CacheEntry` represents individual pieces of cached data.
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
||||
```typescript
|
||||
import { LevelCache, CacheEntry } from '@push.rocks/levelcache';
|
||||
```
|
||||
|
||||
#### Initialization with Optional Configurations
|
||||
|
||||
To create a cache, instantiate the `LevelCache` class with desired configurations. You can specify the limits for memory and disk storage, setup S3 configurations if needed, and more.
|
||||
|
||||
```typescript
|
||||
const myCache = new LevelCache({
|
||||
cacheId: 'myUniqueCacheId', // Unique ID for cache delineation
|
||||
maxMemoryStorageInMB: 10, // Maximum memory use in MB (default 0.5 MB)
|
||||
maxDiskStorageInMB: 100, // Maximum disk space in MB (default 10 MB)
|
||||
diskStoragePath: './myCache', // Path for storing disk cache; default is '.nogit'
|
||||
s3Config: {
|
||||
accessKeyId: 'yourAccessKeyId', // AWS S3 access key
|
||||
secretAccessKey: 'yourSecretAccessKey', // Corresponding secret key
|
||||
region: 'us-west-2' // AWS region, e.g., 'us-west-2'
|
||||
},
|
||||
s3BucketName: 'myBucketName', // Designated name for S3 bucket
|
||||
immutableCache: false, // Whether stored cache entries should remain unaltered
|
||||
persistentCache: true, // Should the cache persist upon restarts
|
||||
});
|
||||
```
|
||||
|
||||
### 3. Storing and Retrieving Data
|
||||
|
||||
`LevelCache` methods enable seamless data storage and retrieval, handling complexity under the hood.
|
||||
|
||||
#### Storing Data
|
||||
|
||||
Create a `CacheEntry` specifying the data content and time-to-live (`ttl`). Use `storeCacheEntryByKey` to add this entry to the cache.
|
||||
|
||||
```typescript
|
||||
async function storeData() {
|
||||
// Wait for cache to be ready before operations
|
||||
await myCache.ready;
|
||||
|
||||
const entryContents = Buffer.from('Caching this data');
|
||||
const myCacheEntry = new CacheEntry({
|
||||
ttl: 7200000, // Time-to-live in milliseconds (2 hours)
|
||||
contents: entryContents,
|
||||
});
|
||||
|
||||
// Storing the cache entry associated with a specific key
|
||||
await myCache.storeCacheEntryByKey('someDataKey', myCacheEntry);
|
||||
}
|
||||
```
|
||||
|
||||
#### Retrieving Data
|
||||
|
||||
Retrieve stored data using `retrieveCacheEntryByKey`. The retrieved `CacheEntry` will give access to the original data.
|
||||
|
||||
```typescript
|
||||
async function retrieveData() {
|
||||
const retrievedEntry = await myCache.retrieveCacheEntryByKey('someDataKey');
|
||||
if (retrievedEntry) {
|
||||
const data = retrievedEntry.contents.toString();
|
||||
console.log(data); // Expected output: Caching this data
|
||||
} else {
|
||||
console.log('Data not found or expired.');
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Key Management: Updating and Deleting
|
||||
|
||||
#### Deleting Cache Entries
|
||||
|
||||
Remove entries with `deleteCacheEntryByKey`, enabling clean cache management.
|
||||
|
||||
```typescript
|
||||
async function deleteData() {
|
||||
// Removes an entry using its unique key identifier
|
||||
await myCache.deleteCacheEntryByKey('someDataKey');
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Cache Cleaning
|
||||
|
||||
Often, managing storage limits or removing outdated data becomes essential. The library supports these scenarios.
|
||||
|
||||
#### Automated Cleaning
|
||||
|
||||
While cache entries will naturally expire with `ttl` values, you can force-remove outdated entries.
|
||||
|
||||
```typescript
|
||||
// Clean outdated or expired entries
|
||||
await myCache.cleanOutdated();
|
||||
```
|
||||
|
||||
#### Full Cache Reset
|
||||
|
||||
Clear all entries, efficiently resetting your cache storage.
|
||||
|
||||
```typescript
|
||||
// Flush entire cache content
|
||||
await myCache.cleanAll();
|
||||
```
|
||||
|
||||
### 6. Configuring and Managing Advanced Use Cases
|
||||
|
||||
The flexible nature of `@push.rocks/levelcache` grants additional customization suited for more advanced requirements.
|
||||
|
||||
#### Custom Route Management
|
||||
|
||||
For certain demands, you might want to specify distinct data handling policies or routing logic.
|
||||
|
||||
- Adjust S3 handling, size thresholds, or immutability options dynamically.
|
||||
- Utilize internal API expansions defined within the library for fine-grained operations.
|
||||
|
||||
#### Handling Large Datasets
|
||||
|
||||
Tailor the cache levels (memory, disk, S3) to accommodate higher loads:
|
||||
|
||||
```typescript
|
||||
const largeDatasetCache = new LevelCache({
|
||||
cacheId: 'largeDatasetCache',
|
||||
// Customize limits and behavior for particular patterns
|
||||
maxMemoryStorageInMB: 1024, // 1 GB memory allocation
|
||||
maxDiskStorageInMB: 2048, // 2 GB disk space allowance
|
||||
maxS3StorageInMB: 10240, // 10 GB S3 backup buffering
|
||||
});
|
||||
```
|
||||
|
||||
With intelligent routing and management embedded, `LevelCache` ensures optimal trade-offs between speed and stability.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
36
test/test.ts
36
test/test.ts
@ -1,15 +1,43 @@
|
||||
import { expect, tap } from '@pushrocks/tapbundle';
|
||||
import * as levelcache from '../ts/index';
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as levelcache from '../ts/index.js';
|
||||
import { CacheEntry } from '../ts/index.js';
|
||||
|
||||
let testLevelCache: levelcache.LevelCache;
|
||||
|
||||
tap.test('should create a new levelcache instance', async () => {
|
||||
testLevelCache = new levelcache.LevelCache();
|
||||
expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
|
||||
testLevelCache = new levelcache.LevelCache({
|
||||
cacheId: 'myCache',
|
||||
});
|
||||
expect(testLevelCache).toBeInstanceOf(levelcache.LevelCache);
|
||||
});
|
||||
|
||||
tap.test('should cache a value', async () => {
|
||||
await testLevelCache.storeCacheEntryByKey(
|
||||
'mykey',
|
||||
new CacheEntry({
|
||||
contents: Buffer.from('heythere'),
|
||||
ttl: 10000,
|
||||
typeInfo: 'string',
|
||||
})
|
||||
);
|
||||
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
|
||||
expect(result.contents.toString()).toEqual('heythere');
|
||||
});
|
||||
|
||||
tap.test('should respect ttl', async (tools) => {
|
||||
await testLevelCache.storeCacheEntryByKey(
|
||||
'mykey',
|
||||
new CacheEntry({
|
||||
contents: Buffer.from('heythere'),
|
||||
ttl: 1000,
|
||||
typeInfo: 'string',
|
||||
})
|
||||
);
|
||||
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
|
||||
expect(result.contents.toString()).toEqual('heythere');
|
||||
await tools!.delayFor(1100);
|
||||
const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey');
|
||||
expect(result2).toBeNull();
|
||||
});
|
||||
|
||||
tap.start();
|
||||
|
8
ts/00_commitinfo_data.ts
Normal file
8
ts/00_commitinfo_data.ts
Normal file
@ -0,0 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @push.rocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/levelcache',
|
||||
version: '3.1.1',
|
||||
description: 'A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.'
|
||||
}
|
@ -1 +1,2 @@
|
||||
export * from './levelcache.classes.levelcache';
|
||||
export * from './levelcache.classes.levelcache.js';
|
||||
export * from './levelcache.classes.cacheentry.js';
|
||||
|
38
ts/levelcache.abstract.classes.cache.ts
Normal file
38
ts/levelcache.abstract.classes.cache.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
|
||||
export abstract class AbstractCache {
|
||||
public abstract ready: Promise<void>;
|
||||
public abstract status: 'active' | 'inactive';
|
||||
|
||||
// Cache Entries
|
||||
/**
|
||||
* store a Blob
|
||||
*/
|
||||
public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
|
||||
|
||||
/**
|
||||
* retrieve cache entry
|
||||
*/
|
||||
public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
|
||||
|
||||
/**
|
||||
* checks for the presence of a key
|
||||
* @param keyArg
|
||||
*/
|
||||
public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* delete a key
|
||||
*/
|
||||
public abstract deleteCacheEntryByKey(keyArg: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* clean the cache
|
||||
*/
|
||||
public abstract cleanOutdated(): Promise<void>;
|
||||
|
||||
/**
|
||||
* cleans the complete cache
|
||||
*/
|
||||
public abstract cleanAll(): Promise<void>;
|
||||
}
|
@ -1,8 +1,71 @@
|
||||
import * as plugins from './levelcache.plugins';
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
import * as paths from './levelcache.paths.js';
|
||||
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
|
||||
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
export class CacheDiskManager {
|
||||
export class CacheDiskManager extends AbstractCache {
|
||||
private levelCacheRef: LevelCache;
|
||||
private readyDeferred = plugins.smartpromise.defer<void>();
|
||||
|
||||
public ready = this.readyDeferred.promise;
|
||||
public status: 'active' | 'inactive';
|
||||
public fsPath: string;
|
||||
public maxCacheSizeInMb: number;
|
||||
|
||||
constructor(levelCacheRefArg: LevelCache) {
|
||||
super();
|
||||
this.levelCacheRef = levelCacheRefArg;
|
||||
this.init();
|
||||
}
|
||||
|
||||
public async init() {
|
||||
if (this.levelCacheRef.options.diskStoragePath) {
|
||||
this.fsPath = plugins.path.join(
|
||||
this.levelCacheRef.options.diskStoragePath,
|
||||
this.levelCacheRef.options.cacheId
|
||||
);
|
||||
} else {
|
||||
this.fsPath = plugins.path.join(paths.nogitDir, this.levelCacheRef.options.cacheId);
|
||||
}
|
||||
if (this.status === 'active') {
|
||||
plugins.smartfile.fs.ensureDirSync(this.fsPath);
|
||||
}
|
||||
this.readyDeferred.resolve();
|
||||
}
|
||||
|
||||
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
|
||||
const fileString = await plugins.smartfile.fs.toStringSync(
|
||||
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
|
||||
);
|
||||
return CacheEntry.fromStorageJsonString(fileString);
|
||||
}
|
||||
|
||||
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
|
||||
await plugins.smartfile.memory.toFs(
|
||||
cacheEntryArg.foldToJson(),
|
||||
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
|
||||
);
|
||||
}
|
||||
|
||||
public async checkKeyPresence(keyArg: string): Promise<boolean> {
|
||||
return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
|
||||
}
|
||||
|
||||
public async deleteCacheEntryByKey(keyArg: string) {
|
||||
await plugins.smartfile.fs.remove(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
|
||||
}
|
||||
|
||||
public async cleanOutdated() {}
|
||||
|
||||
public async cleanAll() {
|
||||
if (this.status === 'active') {
|
||||
if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
|
||||
await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,5 +1,50 @@
|
||||
import * as plugins from './levelcache.plugins';
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
|
||||
|
||||
export class CacheMemoryManager {
|
||||
export class CacheMemoryManager extends AbstractCache {
|
||||
private levelCacheRef: LevelCache;
|
||||
private fastMap = new plugins.lik.FastMap<CacheEntry>();
|
||||
private readyDeferred = plugins.smartpromise.defer<void>();
|
||||
|
||||
public ready = this.readyDeferred.promise;
|
||||
public status: 'active' | 'inactive';
|
||||
|
||||
constructor(levelCacheRefArg: LevelCache) {
|
||||
super();
|
||||
this.levelCacheRef = levelCacheRefArg;
|
||||
this.init();
|
||||
}
|
||||
|
||||
public async init() {
|
||||
this.status = 'active';
|
||||
this.readyDeferred.resolve();
|
||||
}
|
||||
|
||||
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
|
||||
this.fastMap.addToMap(keyArg, cacheEntryArg, { force: true });
|
||||
}
|
||||
|
||||
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
|
||||
return this.fastMap.getByKey(keyArg);
|
||||
}
|
||||
|
||||
public async checkKeyPresence(keyArg: string): Promise<boolean> {
|
||||
if (this.fastMap.getByKey(keyArg)) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async deleteCacheEntryByKey(keyArg: string) {
|
||||
this.fastMap.removeFromMap(keyArg);
|
||||
}
|
||||
|
||||
public async cleanOutdated() {}
|
||||
|
||||
public async cleanAll() {
|
||||
this.fastMap.clean();
|
||||
}
|
||||
}
|
@ -1,8 +1,87 @@
|
||||
import * as plugins from './levelcache.plugins';
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
|
||||
import { LevelCache } from './levelcache.classes.levelcache.js';
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
export class CacheS3Manager {
|
||||
export class CacheS3Manager extends AbstractCache {
|
||||
private levelCacheRef: LevelCache;
|
||||
private smartbucket: plugins.smartbucket.SmartBucket;
|
||||
private s3CacheBucket: plugins.smartbucket.Bucket;
|
||||
private s3CacheDir: plugins.smartbucket.Directory;
|
||||
private readyDeferred = plugins.smartpromise.defer<void>();
|
||||
|
||||
public ready = this.readyDeferred.promise;
|
||||
public status: 'active' | 'inactive';
|
||||
|
||||
constructor(levelCacheRefArg: LevelCache) {
|
||||
super();
|
||||
this.levelCacheRef = levelCacheRefArg;
|
||||
this.init();
|
||||
}
|
||||
|
||||
public async init() {
|
||||
if (this.levelCacheRef.options.s3Config) {
|
||||
this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
|
||||
this.s3CacheBucket = await this.smartbucket.getBucketByName('');
|
||||
this.s3CacheDir = await (
|
||||
await this.s3CacheBucket.getBaseDirectory()
|
||||
).getSubDirectoryByName(this.levelCacheRef.options.cacheId);
|
||||
if (this.levelCacheRef.options.maxS3StorageInMB) {
|
||||
console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
|
||||
} else {
|
||||
console.log(`s3 cache started without limit. Automatically applying timebox of 1 month`);
|
||||
}
|
||||
this.status = 'active';
|
||||
} else {
|
||||
this.status = 'inactive';
|
||||
}
|
||||
this.readyDeferred.resolve();
|
||||
}
|
||||
|
||||
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
|
||||
const jsonFileString = (await this.s3CacheDir.fastGet({
|
||||
path: encodeURIComponent(keyArg),
|
||||
})).toString();
|
||||
const cacheEntry = CacheEntry.fromStorageJsonString(jsonFileString);
|
||||
return cacheEntry;
|
||||
}
|
||||
|
||||
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
|
||||
await this.s3CacheDir.fastPut({
|
||||
path: encodeURIComponent(keyArg),
|
||||
contents: cacheEntryArg.toStorageJsonString()
|
||||
});
|
||||
}
|
||||
|
||||
public async checkKeyPresence(keyArg: string): Promise<boolean> {
|
||||
const files = await this.s3CacheDir.listFiles();
|
||||
for (const file of files) {
|
||||
if (file.name === keyArg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public async deleteCacheEntryByKey(keyArg: string) {
|
||||
if (this.status === 'active') {
|
||||
await this.s3CacheDir.fastRemove({
|
||||
path: encodeURIComponent(keyArg),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* clean outdated
|
||||
*/
|
||||
public async cleanOutdated() {}
|
||||
|
||||
public async cleanAll() {
|
||||
await this.s3CacheDir.delete({
|
||||
mode: 'permanent',
|
||||
});
|
||||
}
|
||||
}
|
@ -1,5 +1,44 @@
|
||||
import * as plugins from './levelcache.plugins';
|
||||
|
||||
export class CacheEntry {
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
|
||||
export interface ICacheEntryConstructorOptions {
|
||||
key?: string;
|
||||
ttl: number;
|
||||
typeInfo?: string;
|
||||
contents: Buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* a CacheEntry
|
||||
*/
|
||||
export class CacheEntry
|
||||
extends plugins.smartjson.Smartjson
|
||||
implements ICacheEntryConstructorOptions
|
||||
{
|
||||
public static fromStorageJsonString(storageJsonString: string) {
|
||||
return new CacheEntry(plugins.smartjson.parse(storageJsonString));
|
||||
}
|
||||
|
||||
@plugins.smartjson.foldDec()
|
||||
public key: string;
|
||||
|
||||
@plugins.smartjson.foldDec()
|
||||
public ttl: number;
|
||||
|
||||
@plugins.smartjson.foldDec()
|
||||
public createdAt: number;
|
||||
|
||||
@plugins.smartjson.foldDec()
|
||||
public typeInfo: string;
|
||||
|
||||
@plugins.smartjson.foldDec()
|
||||
contents: Buffer;
|
||||
|
||||
public toStorageJsonString(): string {
|
||||
return this.foldToJson();
|
||||
}
|
||||
|
||||
constructor(optionsArg: ICacheEntryConstructorOptions) {
|
||||
super();
|
||||
Object.assign(this, optionsArg);
|
||||
}
|
||||
}
|
75
ts/levelcache.classes.cacherouter.ts
Normal file
75
ts/levelcache.classes.cacherouter.ts
Normal file
@ -0,0 +1,75 @@
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
import { LevelCache } from './levelcache.classes.levelcache.js';
|
||||
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
|
||||
export class CacheRouter {
|
||||
public levelCacheRef: LevelCache;
|
||||
public cacheKeyMap = new plugins.lik.FastMap<AbstractCache>();
|
||||
|
||||
constructor(levelCacheRef: LevelCache) {
|
||||
this.levelCacheRef = levelCacheRef;
|
||||
}
|
||||
|
||||
/**
|
||||
* gets the relevant cache to perform a store action on
|
||||
*/
|
||||
async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
|
||||
let returnCache: AbstractCache;
|
||||
const mbToBytesMultiplier = 1000 * 1000;
|
||||
const maxMemoryBytes = this.levelCacheRef.options.maxMemoryStorageInMB * mbToBytesMultiplier;
|
||||
const maxDiskBytes = this.levelCacheRef.options.maxDiskStorageInMB * mbToBytesMultiplier;
|
||||
const maxS3Bytes = this.levelCacheRef.options.maxS3StorageInMB * mbToBytesMultiplier;
|
||||
|
||||
switch (true) {
|
||||
case cacheEntry.contents.byteLength <= maxMemoryBytes &&
|
||||
this.levelCacheRef.cacheMemoryManager.status === 'active':
|
||||
returnCache = this.levelCacheRef.cacheMemoryManager;
|
||||
break;
|
||||
case this.levelCacheRef.cacheDiskManager.status === 'active' &&
|
||||
cacheEntry.contents.byteLength > maxMemoryBytes &&
|
||||
cacheEntry.contents.byteLength <= maxDiskBytes &&
|
||||
this.levelCacheRef.cacheDiskManager.status === 'active':
|
||||
returnCache = this.levelCacheRef.cacheDiskManager;
|
||||
break;
|
||||
case cacheEntry.contents.byteLength > maxDiskBytes &&
|
||||
cacheEntry.contents.byteLength < maxS3Bytes &&
|
||||
this.levelCacheRef.cacheS3Manager.status === 'active':
|
||||
returnCache = this.levelCacheRef.cacheS3Manager;
|
||||
break;
|
||||
default:
|
||||
returnCache = null;
|
||||
}
|
||||
this.cacheKeyMap.addToMap(keyArg, returnCache);
|
||||
return returnCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* gets the relevant cache to perform a retrieval action on
|
||||
*/
|
||||
async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
|
||||
const done = plugins.smartpromise.defer<AbstractCache>();
|
||||
const returnCache = this.cacheKeyMap.getByKey(keyArg);
|
||||
if (!returnCache && this.levelCacheRef.options.persistentCache) {
|
||||
const checkCache = (cacheArg: AbstractCache) => {
|
||||
const resultPromise = cacheArg.checkKeyPresence(keyArg);
|
||||
resultPromise.then((hasKeyArg) => {
|
||||
if (hasKeyArg) {
|
||||
done.resolve(cacheArg);
|
||||
}
|
||||
});
|
||||
return resultPromise;
|
||||
};
|
||||
Promise.all([
|
||||
checkCache(this.levelCacheRef.cacheMemoryManager),
|
||||
checkCache(this.levelCacheRef.cacheDiskManager),
|
||||
checkCache(this.levelCacheRef.cacheMemoryManager),
|
||||
]).then(() => {
|
||||
done.resolve(returnCache);
|
||||
});
|
||||
} else {
|
||||
done.resolve(returnCache);
|
||||
}
|
||||
return done.promise;
|
||||
}
|
||||
}
|
@ -1,42 +1,121 @@
|
||||
import * as plugins from './levelcache.plugins';
|
||||
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager';
|
||||
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
|
||||
import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager.js';
|
||||
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager.js';
|
||||
import { CacheS3Manager } from './levelcache.classes.cache.s3manager.js';
|
||||
import { CacheEntry } from './levelcache.classes.cacheentry.js';
|
||||
import { CacheRouter } from './levelcache.classes.cacherouter.js';
|
||||
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
|
||||
|
||||
export interface ILevelCacheConstructorOptions {
|
||||
/**
|
||||
* a unique id when having more than one cache
|
||||
*/
|
||||
cacheId: string;
|
||||
maxMemoryStorageInMB?: number;
|
||||
maxDiskStorageInMB?: number;
|
||||
maxS3StorageInMB?: number;
|
||||
diskStoragePath?: string;
|
||||
s3Config?: plugins.tsclass.storage.IS3Descriptor;
|
||||
s3BucketName?: string;
|
||||
forceLevel?: 'memory' | 'disk' | 's3';
|
||||
expirationInMs?: number;
|
||||
immutableCache?: boolean;
|
||||
persistentCache?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* a leveled cache for storing things for a short time
|
||||
*/
|
||||
export class LevelCache {
|
||||
public cacheMap = new plugins.lik.Objectmap();
|
||||
public cacheDiskManager = new CacheDiskManager();
|
||||
public cacheMemoryManager = new CacheMemoryManager();
|
||||
public cacheS3Manager = new CacheS3Manager();
|
||||
export class LevelCache extends AbstractCache {
|
||||
private readyDeferred = plugins.smartpromise.defer<void>();
|
||||
|
||||
// Blobs
|
||||
/**
|
||||
* store a Blob
|
||||
*/
|
||||
public async storeBlobByKey () {};
|
||||
public ready = this.readyDeferred.promise;
|
||||
|
||||
/**
|
||||
* retrieve a blob
|
||||
*/
|
||||
public async retrieveBlob () {};
|
||||
public status: 'active' = 'active'; // artifact of AbstractCache
|
||||
|
||||
public cacheRouter = new CacheRouter(this);
|
||||
public cacheDiskManager: CacheDiskManager;
|
||||
public cacheMemoryManager: CacheMemoryManager;
|
||||
public cacheS3Manager: CacheS3Manager;
|
||||
|
||||
// Cache Entries
|
||||
/**
|
||||
* store a Cache Entries
|
||||
*/
|
||||
public async storeCacheEntry() {};
|
||||
public options: ILevelCacheConstructorOptions;
|
||||
|
||||
constructor(optionsArg: ILevelCacheConstructorOptions) {
|
||||
super();
|
||||
this.options = {
|
||||
maxMemoryStorageInMB: 0.5,
|
||||
maxDiskStorageInMB: 10,
|
||||
maxS3StorageInMB: 50,
|
||||
...optionsArg,
|
||||
};
|
||||
this.init();
|
||||
}
|
||||
|
||||
public async init() {
|
||||
this.cacheMemoryManager = new CacheMemoryManager(this);
|
||||
this.cacheDiskManager = new CacheDiskManager(this);
|
||||
this.cacheS3Manager = new CacheS3Manager(this);
|
||||
await Promise.all([
|
||||
this.cacheDiskManager.ready,
|
||||
this.cacheMemoryManager.ready,
|
||||
this.cacheS3Manager.ready,
|
||||
]);
|
||||
this.readyDeferred.resolve();
|
||||
}
|
||||
|
||||
// store things
|
||||
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
|
||||
cacheEntryArg.key = keyArg;
|
||||
const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
|
||||
cacheEntryArg.createdAt = Date.now();
|
||||
await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
|
||||
}
|
||||
|
||||
// retrieve things
|
||||
/**
|
||||
* retrieve cache entry
|
||||
*/
|
||||
public async retrieveCacheEntry () {};
|
||||
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
|
||||
const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
|
||||
if (targetCache) {
|
||||
const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
|
||||
if (cacheEntry.createdAt + cacheEntry.ttl < Date.now()) {
|
||||
await this.deleteCacheEntryByKey(keyArg).catch();
|
||||
return null;
|
||||
}
|
||||
return cacheEntry;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async checkKeyPresence(keyArg: string): Promise<boolean> {
|
||||
return plugins.smartpromise.getFirstTrueOrFalse([
|
||||
this.cacheMemoryManager.checkKeyPresence(keyArg),
|
||||
this.cacheDiskManager.checkKeyPresence(keyArg),
|
||||
this.cacheS3Manager.checkKeyPresence(keyArg),
|
||||
]);
|
||||
}
|
||||
|
||||
public async deleteCacheEntryByKey(keyArg) {
|
||||
await Promise.all([
|
||||
this.cacheMemoryManager.deleteCacheEntryByKey(keyArg),
|
||||
this.cacheDiskManager.deleteCacheEntryByKey(keyArg),
|
||||
this.cacheS3Manager.deleteCacheEntryByKey(keyArg),
|
||||
]);
|
||||
}
|
||||
|
||||
// cache maintenance
|
||||
public async cleanOutdated() {}
|
||||
|
||||
/**
|
||||
* cleans the cache
|
||||
* cleans the complete cache
|
||||
*/
|
||||
public clean() {};
|
||||
public async cleanAll(): Promise<void> {
|
||||
await Promise.all([
|
||||
this.cacheDiskManager.cleanAll(),
|
||||
this.cacheDiskManager.cleanAll(),
|
||||
this.cacheS3Manager.cleanAll(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
@ -0,0 +1,7 @@
|
||||
import * as plugins from './levelcache.plugins.js';
|
||||
|
||||
export const packageDir = plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../'
|
||||
);
|
||||
export const nogitDir = plugins.path.join(packageDir, '.nogit/');
|
@ -1,21 +1,36 @@
|
||||
// node native scope
|
||||
import * as path from 'path';
|
||||
|
||||
export {
|
||||
path
|
||||
}
|
||||
export { path };
|
||||
|
||||
// @pushrocks scope
|
||||
import * as lik from '@pushrocks/lik';
|
||||
import * as smartbucket from '@pushrocks/smartbucket';
|
||||
import * as smartcache from '@pushrocks/smartcache';
|
||||
import * as smartfile from '@pushrocks/smartfile';
|
||||
import * as smartstring from '@pushrocks/smartstring';
|
||||
// @push.rocks scope
|
||||
import * as lik from '@push.rocks/lik';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import * as smartcache from '@push.rocks/smartcache';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartjson from '@push.rocks/smartjson';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartstring from '@push.rocks/smartstring';
|
||||
import * as smartunique from '@push.rocks/smartunique';
|
||||
import * as taskbuffer from '@push.rocks/taskbuffer';
|
||||
|
||||
export {
|
||||
lik,
|
||||
smartbucket,
|
||||
smartcache,
|
||||
smartfile,
|
||||
smartstring
|
||||
smartjson,
|
||||
smartpath,
|
||||
smartpromise,
|
||||
smartstring,
|
||||
smartunique,
|
||||
taskbuffer,
|
||||
};
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export {
|
||||
tsclass,
|
||||
};
|
||||
|
14
tsconfig.json
Normal file
14
tsconfig.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
17
tslint.json
17
tslint.json
@ -1,17 +0,0 @@
|
||||
{
|
||||
"extends": ["tslint:latest", "tslint-config-prettier"],
|
||||
"rules": {
|
||||
"semicolon": [true, "always"],
|
||||
"no-console": false,
|
||||
"ordered-imports": false,
|
||||
"object-literal-sort-keys": false,
|
||||
"member-ordering": {
|
||||
"options":{
|
||||
"order": [
|
||||
"static-method"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"defaultSeverity": "warning"
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user