34 Commits

Author SHA1 Message Date
96080c5de0 3.0.8 2024-02-14 01:33:57 +01:00
a981c63f9b fix(core): update 2024-02-14 01:33:56 +01:00
cd5e2f03f4 3.0.7 2024-02-14 01:32:20 +01:00
eb5c1f40ee fix(core): update 2024-02-14 01:32:19 +01:00
ed1d66ee2a 3.0.6 2023-07-21 00:44:40 +02:00
45122024fc fix(core): update 2023-07-21 00:44:39 +02:00
fcaed1a3a0 3.0.5 2023-07-21 00:44:07 +02:00
0ce5eba798 fix(core): update 2023-07-21 00:44:06 +02:00
9d6111a77f 3.0.4 2023-07-20 22:54:08 +02:00
ac11045812 fix(core): update 2023-07-20 22:54:07 +02:00
2266ee0c73 switch to new org scheme 2023-07-11 00:00:42 +02:00
e4f0a60644 switch to new org scheme 2023-07-10 02:41:30 +02:00
946e1cb7d3 3.0.3 2023-01-09 13:01:09 +01:00
6e5d17a793 fix(core): update 2023-01-09 13:01:08 +01:00
3f7a3c2786 3.0.2 2022-06-09 19:23:56 +02:00
925921a9fe fix(core): update 2022-06-09 19:23:55 +02:00
61ccf38d85 3.0.1 2022-04-04 14:18:39 +02:00
061a79b0fd update 2022-04-04 14:18:36 +02:00
2f4fb82918 3.0.0 2022-04-02 17:40:38 +02:00
266da3b476 2.0.1 2022-03-22 23:15:08 +01:00
074767b294 fix(core): update 2022-03-22 23:15:08 +01:00
85abc4da3d 2.0.0 2022-03-22 22:45:13 +01:00
716d805723 BREAKING CHANGE(core): switch to esm 2022-03-22 22:45:12 +01:00
9feaa921bf 1.0.10 2021-05-10 14:26:32 +00:00
6b57e8b1f3 fix(caching): properly respect ttl for all cache levels 2021-05-10 14:26:32 +00:00
f7b6df5ff7 1.0.9 2021-04-23 18:50:21 +00:00
6a4df84f4f 1.0.8 2021-04-23 18:41:30 +00:00
17b59fd222 fix(core): update 2021-04-23 18:41:30 +00:00
d18f4b6928 1.0.7 2021-04-23 18:40:58 +00:00
e53ebc74f8 fix(core): update 2021-04-23 18:40:57 +00:00
be8227f451 1.0.6 2020-02-15 22:38:29 +00:00
d8f526ef2d fix(core): update 2020-02-15 22:38:28 +00:00
92b858e248 1.0.5 2020-02-14 18:10:10 +00:00
26511fc10e fix(core): update 2020-02-14 18:10:10 +00:00
25 changed files with 6975 additions and 2145 deletions

View File

@ -0,0 +1,66 @@
name: Default (not tags)
on:
push:
tags-ignore:
- '**'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
- name: Run npm prepare
run: npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build

View File

@ -0,0 +1,124 @@
name: Default (tags)
on:
push:
tags:
- '*'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Release
run: |
npmci node install stable
npmci npm publish
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Code quality
run: |
npmci command npm install -g typescript
npmci npm install
- name: Trigger
run: npmci trigger
- name: Build docs and upload artifacts
run: |
npmci node install stable
npmci npm install
pnpm install -g @gitzone/tsdoc
npmci command tsdoc
continue-on-error: true

4
.gitignore vendored
View File

@ -15,8 +15,6 @@ node_modules/
# builds
dist/
dist_web/
dist_serve/
dist_ts_web/
dist_*/
# custom

View File

@ -1,126 +0,0 @@
# gitzone ci_default
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
cache:
paths:
- .npmci_cache/
key: '$CI_BUILD_STAGE'
stages:
- security
- test
- release
- metadata
# ====================
# security stage
# ====================
mirror:
stage: security
script:
- npmci git mirror
tags:
- lossless
- docker
- notpriv
snyk:
image: registry.gitlab.com/hosttoday/ht-docker-node:snyk
stage: security
script:
- npmci npm prepare
- npmci command npm install --ignore-scripts
- npmci command snyk test
tags:
- lossless
- docker
- notpriv
# ====================
# test stage
# ====================
testStable:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- priv
testBuild:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- lossless
- docker
- notpriv
release:
stage: release
script:
- npmci node install stable
- npmci npm publish
only:
- tags
tags:
- lossless
- docker
- notpriv
# ====================
# metadata stage
# ====================
codequality:
stage: metadata
allow_failure: true
script:
- npmci command npm install -g tslint typescript
- npmci npm prepare
- npmci npm install
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
tags:
- lossless
- docker
- priv
trigger:
stage: metadata
script:
- npmci trigger
only:
- tags
tags:
- lossless
- docker
- notpriv
pages:
stage: metadata
script:
- npmci node install lts
- npmci command npm install -g @gitzone/tsdoc
- npmci npm prepare
- npmci npm install
- npmci command tsdoc
tags:
- lossless
- docker
- notpriv
only:
- tags
artifacts:
expire_in: 1 week
paths:
- public
allow_failure: true

24
.vscode/launch.json vendored
View File

@ -2,28 +2,10 @@
"version": "0.2.0",
"configurations": [
{
"name": "current file",
"type": "node",
"command": "npm test",
"name": "Run npm test",
"request": "launch",
"args": [
"${relativeFile}"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
},
{
"name": "test.ts",
"type": "node",
"request": "launch",
"args": [
"test/test.ts"
],
"runtimeArgs": ["-r", "@gitzone/tsrun"],
"cwd": "${workspaceRoot}",
"protocol": "inspector",
"internalConsoleOptions": "openOnSessionStart"
"type": "node-terminal"
}
]
}

View File

@ -15,7 +15,7 @@
"properties": {
"projectType": {
"type": "string",
"enum": ["website", "element", "service", "npm"]
"enum": ["website", "element", "service", "npm", "wcc"]
}
}
}

View File

@ -3,10 +3,10 @@
"projectType": "npm",
"module": {
"githost": "gitlab.com",
"gitscope": "pushrocks",
"gitscope": "push.rocks",
"gitrepo": "levelcache",
"shortDescription": "a cache that uses memory/disk/s3 as backup",
"npmPackagename": "@pushrocks/levelcache",
"description": "a cache that uses memory/disk/s3 as backup",
"npmPackagename": "@push.rocks/levelcache",
"license": "MIT",
"projectDomain": "push.rocks"
}

1883
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,41 +1,54 @@
{
"name": "@pushrocks/levelcache",
"version": "1.0.4",
"name": "@push.rocks/levelcache",
"version": "3.0.8",
"private": false,
"description": "a cache that uses memory/disk/s3 as backup",
"main": "dist/index.js",
"typings": "dist/index.d.ts",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/)",
"build": "(tsbuild)",
"format": "(gitzone format)"
"test": "(tstest test/ --web)",
"build": "(tsbuild --web --allowimplicitany)",
"buildDocs": "tsdoc",
"localPublish": "gitzone commit && pnpm run build && pnpm publish && pnpm publish --access public --registry=\"https://registry.npmjs.org\""
},
"devDependencies": {
"@gitzone/tsbuild": "^2.0.22",
"@gitzone/tstest": "^1.0.15",
"@pushrocks/tapbundle": "^3.0.7",
"@types/node": "^13.7.0",
"tslint": "^6.0.0",
"tslint-config-prettier": "^1.15.0"
"@gitzone/tsbuild": "^2.1.66",
"@gitzone/tsrun": "^1.2.44",
"@gitzone/tstest": "^1.0.77",
"@push.rocks/tapbundle": "^5.0.15",
"@types/node": "^20.11.17"
},
"dependencies": {
"@pushrocks/lik": "^3.0.16",
"@pushrocks/smartbucket": "^1.0.24",
"@pushrocks/smartcache": "^1.0.13",
"@pushrocks/smartfile": "^7.0.6",
"@pushrocks/smartstring": "^3.0.18"
"@push.rocks/lik": "^6.0.12",
"@push.rocks/smartbucket": "^2.0.4",
"@push.rocks/smartcache": "^1.0.13",
"@push.rocks/smartenv": "^5.0.12",
"@push.rocks/smartexit": "^1.0.21",
"@push.rocks/smartfile": "^11.0.4",
"@push.rocks/smartjson": "^5.0.10",
"@push.rocks/smartpath": "^5.0.11",
"@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartstring": "^4.0.13",
"@push.rocks/smartunique": "^3.0.6",
"@push.rocks/taskbuffer": "^3.1.7",
"@tsclass/tsclass": "^4.0.51"
},
"files": [
"ts/**/*",
"ts_web/**/*",
"dist/**/*",
"dist_web/**/*",
"dist_*/**/*",
"dist_ts/**/*",
"dist_ts_web/**/*",
"assets/**/*",
"cli.js",
"npmextra.json",
"readme.md"
],
"browserslist": [
"last 1 chrome versions"
]
}

6180
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,23 +1,32 @@
# @pushrocks/levelcache
# @push.rocks/levelcache
a cache that uses memory/disk/s3 as backup
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/levelcache)
* [gitlab.com (source)](https://gitlab.com/pushrocks/levelcache)
* [github.com (source mirror)](https://github.com/pushrocks/levelcache)
* [docs (typedoc)](https://pushrocks.gitlab.io/levelcache/)
* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/levelcache)
* [gitlab.com (source)](https://gitlab.com/push.rocks/levelcache)
* [github.com (source mirror)](https://github.com/push.rocks/levelcache)
* [docs (typedoc)](https://push.rocks.gitlab.io/levelcache/)
## Status for master
[![pipeline status](https://gitlab.com/pushrocks/levelcache/badges/master/pipeline.svg)](https://gitlab.com/pushrocks/levelcache/commits/master)
[![coverage report](https://gitlab.com/pushrocks/levelcache/badges/master/coverage.svg)](https://gitlab.com/pushrocks/levelcache/commits/master)
[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/levelcache.svg)](https://www.npmjs.com/package/@pushrocks/levelcache)
[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/levelcache/badge.svg)](https://snyk.io/test/npm/@pushrocks/levelcache)
[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
Status Category | Status Badge
-- | --
GitLab Pipelines | [![pipeline status](https://gitlab.com/push.rocks/levelcache/badges/master/pipeline.svg)](https://lossless.cloud)
GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/push.rocks/levelcache/badges/master/coverage.svg)](https://lossless.cloud)
npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/levelcache)](https://lossless.cloud)
Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/levelcache)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/levelcache)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/levelcache)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/levelcache)](https://lossless.cloud)
## Usage
@pushrocks/levelcache implements an efficient way to cache larger amounts of data ind a multilevel approach.
smaller and prioritized data is safed in memory.
Use TypeScript for best in class intellisense.
## Contribution
@ -26,7 +35,6 @@ We are always happy for code contributions. If you are not the code contributing
For further information read the linked docs at the top of this readme.
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
## Legal
> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)

View File

@ -1,15 +1,43 @@
import { expect, tap } from '@pushrocks/tapbundle';
import * as levelcache from '../ts/index';
import { expect, tap } from '@push.rocks/tapbundle';
import * as levelcache from '../ts/index.js';
import { CacheEntry } from '../ts/index.js';
let testLevelCache: levelcache.LevelCache;
tap.test('should create a new levelcache instance', async () => {
testLevelCache = new levelcache.LevelCache();
expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
testLevelCache = new levelcache.LevelCache({
cacheId: 'myCache',
});
expect(testLevelCache).toBeInstanceOf(levelcache.LevelCache);
});
tap.test('should cache a value', async () => {
await testLevelCache.storeCacheEntryByKey(
'mykey',
new CacheEntry({
contents: Buffer.from('heythere'),
ttl: 10000,
typeInfo: 'string',
})
);
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result.contents.toString()).toEqual('heythere');
});
tap.test('should respect ttl', async (tools) => {
await testLevelCache.storeCacheEntryByKey(
'mykey',
new CacheEntry({
contents: Buffer.from('heythere'),
ttl: 1000,
typeInfo: 'string',
})
);
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result.contents.toString()).toEqual('heythere');
await tools.delayFor(1100);
const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result2).toBeNull();
});
tap.start();

8
ts/00_commitinfo_data.ts Normal file
View File

@ -0,0 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/levelcache',
version: '3.0.8',
description: 'a cache that uses memory/disk/s3 as backup'
}

View File

@ -1 +1,2 @@
export * from './levelcache.classes.levelcache';
export * from './levelcache.classes.levelcache.js';
export * from './levelcache.classes.cacheentry.js';

View File

@ -0,0 +1,38 @@
import { CacheEntry } from './levelcache.classes.cacheentry.js';
export abstract class AbstractCache {
public abstract ready: Promise<void>;
public abstract status: 'active' | 'inactive';
// Cache Entries
/**
* store a Blob
*/
public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
/**
* retrieve cache entry
*/
public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
/**
* checks for the presence of a key
* @param keyArg
*/
public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
/**
* delete a key
*/
public abstract deleteCacheEntryByKey(keyArg: string): Promise<void>;
/**
* clean the cache
*/
public abstract cleanOutdated(): Promise<void>;
/**
* cleans the complete cache
*/
public abstract cleanAll(): Promise<void>;
}

View File

@ -1,8 +1,71 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import * as paths from './levelcache.paths.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
/**
*
*
*/
export class CacheDiskManager {
export class CacheDiskManager extends AbstractCache {
private levelCacheRef: LevelCache;
private readyDeferred = plugins.smartpromise.defer<void>();
}
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
public fsPath: string;
public maxCacheSizeInMb: number;
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.diskStoragePath) {
this.fsPath = plugins.path.join(
this.levelCacheRef.options.diskStoragePath,
this.levelCacheRef.options.cacheId
);
} else {
this.fsPath = plugins.path.join(paths.nogitDir, this.levelCacheRef.options.cacheId);
}
if (this.status === 'active') {
plugins.smartfile.fs.ensureDirSync(this.fsPath);
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const fileString = await plugins.smartfile.fs.toStringSync(
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
);
return CacheEntry.fromStorageJsonString(fileString);
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await plugins.smartfile.memory.toFs(
cacheEntryArg.foldToJson(),
plugins.path.join(this.fsPath, encodeURIComponent(keyArg))
);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
public async deleteCacheEntryByKey(keyArg: string) {
await plugins.smartfile.fs.remove(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
public async cleanOutdated() {}
public async cleanAll() {
if (this.status === 'active') {
if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);
}
}
}
}

View File

@ -1,5 +1,50 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
export class CacheMemoryManager {
}
export class CacheMemoryManager extends AbstractCache {
private levelCacheRef: LevelCache;
private fastMap = new plugins.lik.FastMap<CacheEntry>();
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
this.status = 'active';
this.readyDeferred.resolve();
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
this.fastMap.addToMap(keyArg, cacheEntryArg, { force: true });
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
return this.fastMap.getByKey(keyArg);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
if (this.fastMap.getByKey(keyArg)) {
return true;
} else {
return false;
}
}
public async deleteCacheEntryByKey(keyArg: string) {
this.fastMap.removeFromMap(keyArg);
}
public async cleanOutdated() {}
public async cleanAll() {
this.fastMap.clean();
}
}

View File

@ -1,8 +1,81 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { LevelCache } from './levelcache.classes.levelcache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
/**
*
*
*/
export class CacheS3Manager {
export class CacheS3Manager extends AbstractCache {
private levelCacheRef: LevelCache;
private smartbucket: plugins.smartbucket.SmartBucket;
private s3CacheBucket: plugins.smartbucket.Bucket;
private s3CacheDir: plugins.smartbucket.Directory;
private readyDeferred = plugins.smartpromise.defer<void>();
}
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.s3Config) {
this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
this.s3CacheBucket = await this.smartbucket.getBucketByName('');
this.s3CacheDir = await (
await this.s3CacheBucket.getBaseDirectory()
).getSubDirectoryByName(this.levelCacheRef.options.cacheId);
if (this.levelCacheRef.options.maxS3StorageInMB) {
console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
} else {
console.log(`s3 cache started without limit. Automatically applying timebox of 1 month`);
}
this.status = 'active';
} else {
this.status = 'inactive';
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const jsonFileString = (await this.s3CacheDir.fastGet(encodeURIComponent(keyArg))).toString();
const cacheEntry = CacheEntry.fromStorageJsonString(jsonFileString);
return cacheEntry;
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await this.s3CacheDir.fastStore(
encodeURIComponent(keyArg),
cacheEntryArg.toStorageJsonString()
);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
const files = await this.s3CacheDir.listFiles();
for (const file of files) {
if (file.name === keyArg) {
return true;
}
}
return false;
}
public async deleteCacheEntryByKey(keyArg: string) {
if (this.status === 'active') {
await this.s3CacheDir.fastRemove(encodeURIComponent(keyArg));
}
}
/**
* clean outdated
*/
public async cleanOutdated() {}
public async cleanAll() {
await this.s3CacheDir.deleteWithAllContents();
}
}

View File

@ -1,5 +1,44 @@
import * as plugins from './levelcache.plugins';
import * as plugins from './levelcache.plugins.js';
export class CacheEntry {
}
export interface ICacheEntryConstructorOptions {
key?: string;
ttl: number;
typeInfo?: string;
contents: Buffer;
}
/**
* a CacheEntry
*/
export class CacheEntry
extends plugins.smartjson.Smartjson
implements ICacheEntryConstructorOptions
{
public static fromStorageJsonString(storageJsonString: string) {
return new CacheEntry(plugins.smartjson.parse(storageJsonString));
}
@plugins.smartjson.foldDec()
public key: string;
@plugins.smartjson.foldDec()
public ttl: number;
@plugins.smartjson.foldDec()
public createdAt: number;
@plugins.smartjson.foldDec()
public typeInfo: string;
@plugins.smartjson.foldDec()
contents: Buffer;
public toStorageJsonString(): string {
return this.foldToJson();
}
constructor(optionsArg: ICacheEntryConstructorOptions) {
super();
Object.assign(this, optionsArg);
}
}

View File

@ -0,0 +1,75 @@
import * as plugins from './levelcache.plugins.js';
import { LevelCache } from './levelcache.classes.levelcache.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
export class CacheRouter {
public levelCacheRef: LevelCache;
public cacheKeyMap = new plugins.lik.FastMap<AbstractCache>();
constructor(levelCacheRef: LevelCache) {
this.levelCacheRef = levelCacheRef;
}
/**
* gets the relevant cache to perform a store action on
*/
async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
let returnCache: AbstractCache;
const mbToBytesMultiplier = 1000 * 1000;
const maxMemoryBytes = this.levelCacheRef.options.maxMemoryStorageInMB * mbToBytesMultiplier;
const maxDiskBytes = this.levelCacheRef.options.maxDiskStorageInMB * mbToBytesMultiplier;
const maxS3Bytes = this.levelCacheRef.options.maxS3StorageInMB * mbToBytesMultiplier;
switch (true) {
case cacheEntry.contents.byteLength <= maxMemoryBytes &&
this.levelCacheRef.cacheMemoryManager.status === 'active':
returnCache = this.levelCacheRef.cacheMemoryManager;
break;
case this.levelCacheRef.cacheDiskManager.status === 'active' &&
cacheEntry.contents.byteLength > maxMemoryBytes &&
cacheEntry.contents.byteLength <= maxDiskBytes &&
this.levelCacheRef.cacheDiskManager.status === 'active':
returnCache = this.levelCacheRef.cacheDiskManager;
break;
case cacheEntry.contents.byteLength > maxDiskBytes &&
cacheEntry.contents.byteLength < maxS3Bytes &&
this.levelCacheRef.cacheS3Manager.status === 'active':
returnCache = this.levelCacheRef.cacheS3Manager;
break;
default:
returnCache = null;
}
this.cacheKeyMap.addToMap(keyArg, returnCache);
return returnCache;
}
/**
* gets the relevant cache to perform a retrieval action on
*/
async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
const done = plugins.smartpromise.defer<AbstractCache>();
const returnCache = this.cacheKeyMap.getByKey(keyArg);
if (!returnCache && this.levelCacheRef.options.persistentCache) {
const checkCache = (cacheArg: AbstractCache) => {
const resultPromise = cacheArg.checkKeyPresence(keyArg);
resultPromise.then((hasKeyArg) => {
if (hasKeyArg) {
done.resolve(cacheArg);
}
});
return resultPromise;
};
Promise.all([
checkCache(this.levelCacheRef.cacheMemoryManager),
checkCache(this.levelCacheRef.cacheDiskManager),
checkCache(this.levelCacheRef.cacheMemoryManager),
]).then(() => {
done.resolve(returnCache);
});
} else {
done.resolve(returnCache);
}
return done.promise;
}
}

View File

@ -1,42 +1,121 @@
import * as plugins from './levelcache.plugins';
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager';
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
import * as plugins from './levelcache.plugins.js';
import { CacheDiskManager } from './levelcache.classes.cache.diskmanager.js';
import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager.js';
import { CacheS3Manager } from './levelcache.classes.cache.s3manager.js';
import { CacheEntry } from './levelcache.classes.cacheentry.js';
import { CacheRouter } from './levelcache.classes.cacherouter.js';
import { AbstractCache } from './levelcache.abstract.classes.cache.js';
export interface ILevelCacheConstructorOptions {
/**
* a unique id when having more than one cache
*/
cacheId: string;
maxMemoryStorageInMB?: number;
maxDiskStorageInMB?: number;
maxS3StorageInMB?: number;
diskStoragePath?: string;
s3Config?: plugins.tsclass.storage.IS3Descriptor;
s3BucketName?: string;
forceLevel?: 'memory' | 'disk' | 's3';
expirationInMs?: number;
immutableCache?: boolean;
persistentCache?: boolean;
}
/**
* a leveled cache for storing things for a short time
*/
export class LevelCache {
public cacheMap = new plugins.lik.Objectmap();
public cacheDiskManager = new CacheDiskManager();
public cacheMemoryManager = new CacheMemoryManager();
public cacheS3Manager = new CacheS3Manager();
export class LevelCache extends AbstractCache {
private readyDeferred = plugins.smartpromise.defer<void>();
// Blobs
/**
* store a Blob
*/
public async storeBlobByKey () {};
public ready = this.readyDeferred.promise;
/**
* retrieve a blob
*/
public async retrieveBlob () {};
public status: 'active' = 'active'; // artifact of AbstractCache
public cacheRouter = new CacheRouter(this);
public cacheDiskManager: CacheDiskManager;
public cacheMemoryManager: CacheMemoryManager;
public cacheS3Manager: CacheS3Manager;
// Cache Entries
/**
* store a Cache Entries
*/
public async storeCacheEntry() {};
public options: ILevelCacheConstructorOptions;
constructor(optionsArg: ILevelCacheConstructorOptions) {
super();
this.options = {
maxMemoryStorageInMB: 0.5,
maxDiskStorageInMB: 10,
maxS3StorageInMB: 50,
...optionsArg,
};
this.init();
}
public async init() {
this.cacheMemoryManager = new CacheMemoryManager(this);
this.cacheDiskManager = new CacheDiskManager(this);
this.cacheS3Manager = new CacheS3Manager(this);
await Promise.all([
this.cacheDiskManager.ready,
this.cacheMemoryManager.ready,
this.cacheS3Manager.ready,
]);
this.readyDeferred.resolve();
}
// store things
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
cacheEntryArg.key = keyArg;
const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
cacheEntryArg.createdAt = Date.now();
await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
}
// retrieve things
/**
* retrieve cache entry
*/
public async retrieveCacheEntry () {};
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
if (targetCache) {
const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
if (cacheEntry.createdAt + cacheEntry.ttl < Date.now()) {
await this.deleteCacheEntryByKey(keyArg).catch();
return null;
}
return cacheEntry;
} else {
return null;
}
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
return plugins.smartpromise.getFirstTrueOrFalse([
this.cacheMemoryManager.checkKeyPresence(keyArg),
this.cacheDiskManager.checkKeyPresence(keyArg),
this.cacheS3Manager.checkKeyPresence(keyArg),
]);
}
public async deleteCacheEntryByKey(keyArg) {
await Promise.all([
this.cacheMemoryManager.deleteCacheEntryByKey(keyArg),
this.cacheDiskManager.deleteCacheEntryByKey(keyArg),
this.cacheS3Manager.deleteCacheEntryByKey(keyArg),
]);
}
// cache maintenance
public async cleanOutdated() {}
/**
* cleans the cache
* cleans the complete cache
*/
public clean() {};
public async cleanAll(): Promise<void> {
await Promise.all([
this.cacheDiskManager.cleanAll(),
this.cacheDiskManager.cleanAll(),
this.cacheS3Manager.cleanAll(),
]);
}
}

View File

@ -0,0 +1,7 @@
import * as plugins from './levelcache.plugins.js';
export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);
export const nogitDir = plugins.path.join(packageDir, '.nogit/');

View File

@ -1,21 +1,36 @@
// node native scope
import * as path from 'path';
export {
path
}
export { path };
// @pushrocks scope
import * as lik from '@pushrocks/lik';
import * as smartbucket from '@pushrocks/smartbucket';
import * as smartcache from '@pushrocks/smartcache';
import * as smartfile from '@pushrocks/smartfile';
import * as smartstring from '@pushrocks/smartstring';
// @push.rocks scope
import * as lik from '@push.rocks/lik';
import * as smartbucket from '@push.rocks/smartbucket';
import * as smartcache from '@push.rocks/smartcache';
import * as smartfile from '@push.rocks/smartfile';
import * as smartjson from '@push.rocks/smartjson';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartstring from '@push.rocks/smartstring';
import * as smartunique from '@push.rocks/smartunique';
import * as taskbuffer from '@push.rocks/taskbuffer';
export {
lik,
smartbucket,
smartcache,
smartfile,
smartstring
smartjson,
smartpath,
smartpromise,
smartstring,
smartunique,
taskbuffer,
};
// @tsclass scope
import * as tsclass from '@tsclass/tsclass';
export {
tsclass,
};

14
tsconfig.json Normal file
View File

@ -0,0 +1,14 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}

View File

@ -1,17 +0,0 @@
{
"extends": ["tslint:latest", "tslint-config-prettier"],
"rules": {
"semicolon": [true, "always"],
"no-console": false,
"ordered-imports": false,
"object-literal-sort-keys": false,
"member-ordering": {
"options":{
"order": [
"static-method"
]
}
}
},
"defaultSeverity": "warning"
}