3.0.5

.gitlab-ci.yml

@@ -12,6 +12,9 @@ stages:
   - release
   - metadata
 
+before_script:
+  - npm install -g @shipzone/npmci
+
 # ====================
 # security stage
 # ====================
@@ -36,6 +39,7 @@ auditProductionDependencies:
     - npmci command npm audit --audit-level=high --only=prod --production
   tags:
     - docker
+  allow_failure: true
 
 auditDevDependencies:
   image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
@@ -96,10 +100,9 @@ codequality:
   only:
     - tags
   script:
-    - npmci command npm install -g tslint typescript
+    - npmci command npm install -g typescript
     - npmci npm prepare
     - npmci npm install
-    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
   tags:
     - lossless
     - docker

.vscode/launch.json

@@ -2,28 +2,10 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "current file",
-      "type": "node",
+      "command": "npm test",
+      "name": "Run npm test",
       "request": "launch",
-      "args": [
-        "${relativeFile}"
-      ],
-      "runtimeArgs": ["-r", "@gitzone/tsrun"],
-      "cwd": "${workspaceRoot}",
-      "protocol": "inspector",
-      "internalConsoleOptions": "openOnSessionStart"
-    },
-    {
-      "name": "test.ts",
-      "type": "node",
-      "request": "launch",
-      "args": [
-        "test/test.ts"
-      ],
-      "runtimeArgs": ["-r", "@gitzone/tsrun"],
-      "cwd": "${workspaceRoot}",
-      "protocol": "inspector",
-      "internalConsoleOptions": "openOnSessionStart"
+      "type": "node-terminal"
     }
   ]
 }

npmextra.json

@@ -3,10 +3,10 @@
     "projectType": "npm",
     "module": {
       "githost": "gitlab.com",
-      "gitscope": "pushrocks",
+      "gitscope": "push.rocks",
       "gitrepo": "levelcache",
-      "shortDescription": "a cache that uses memory/disk/s3 as backup",
-      "npmPackagename": "@pushrocks/levelcache",
+      "description": "a cache that uses memory/disk/s3 as backup",
+      "npmPackagename": "@push.rocks/levelcache",
       "license": "MIT",
       "projectDomain": "push.rocks"
     }

package.json

@@ -1,35 +1,37 @@
 {
-  "name": "@pushrocks/levelcache",
-  "version": "1.0.8",
+  "name": "@push.rocks/levelcache",
+  "version": "3.0.5",
   "private": false,
   "description": "a cache that uses memory/disk/s3 as backup",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
+  "type": "module",
   "author": "Lossless GmbH",
   "license": "MIT",
   "scripts": {
     "test": "(tstest test/ --web)",
-    "build": "(tsbuild --web)"
+    "build": "(tsbuild --web --allowimplicitany)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.25",
-    "@gitzone/tstest": "^1.0.52",
-    "@pushrocks/tapbundle": "^3.2.14",
-    "@types/node": "^14.14.41",
-    "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.15.0"
+    "@gitzone/tsbuild": "^2.1.66",
+    "@gitzone/tsrun": "^1.2.44",
+    "@gitzone/tstest": "^1.0.77",
+    "@push.rocks/tapbundle": "^5.0.12",
+    "@types/node": "^20.4.2"
   },
   "dependencies": {
-    "@pushrocks/lik": "^4.0.20",
-    "@pushrocks/smartbucket": "^1.0.39",
-    "@pushrocks/smartcache": "^1.0.13",
-    "@pushrocks/smartexit": "^1.0.15",
-    "@pushrocks/smartfile": "^8.0.9",
-    "@pushrocks/smartjson": "^4.0.6",
-    "@pushrocks/smartpromise": "^3.1.5",
-    "@pushrocks/smartstring": "^3.0.24",
-    "@pushrocks/smartunique": "^3.0.3",
-    "@pushrocks/taskbuffer": "^2.1.13"
+    "@push.rocks/lik": "^6.0.3",
+    "@push.rocks/smartbucket": "^2.0.2",
+    "@push.rocks/smartcache": "^1.0.13",
+    "@push.rocks/smartenv": "^5.0.2",
+    "@push.rocks/smartexit": "^1.0.20",
+    "@push.rocks/smartfile": "^10.0.28",
+    "@push.rocks/smartjson": "^5.0.5",
+    "@push.rocks/smartpath": "^5.0.11",
+    "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartstring": "^4.0.2",
+    "@push.rocks/smartunique": "^3.0.3",
+    "@push.rocks/taskbuffer": "^3.0.1"
   },
   "files": [
     "ts/**/*",

test/test.ts

@@ -1,6 +1,6 @@
-import { expect, tap } from '@pushrocks/tapbundle';
-import * as levelcache from '../ts/index';
-import { CacheEntry } from '../ts/index';
+import { expect, tap } from '@push.rocks/tapbundle';
+import * as levelcache from '../ts/index.js';
+import { CacheEntry } from '../ts/index.js';
 
 let testLevelCache: levelcache.LevelCache;
 
@@ -8,7 +8,7 @@ tap.test('should create a new levelcache instance', async () => {
   testLevelCache = new levelcache.LevelCache({
     cacheId: 'myCache',
   });
-  expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
+  expect(testLevelCache).toBeInstanceOf(levelcache.LevelCache);
 });
 
 tap.test('should cache a value', async () => {
@@ -21,7 +21,23 @@ tap.test('should cache a value', async () => {
     })
   );
   const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
-  console.log(result);
+  expect(result.contents.toString()).toEqual('heythere');
+});
+
+tap.test('should respect ttl', async (tools) => {
+  await testLevelCache.storeCacheEntryByKey(
+    'mykey',
+    new CacheEntry({
+      contents: Buffer.from('heythere'),
+      ttl: 1000,
+      typeInfo: 'string',
+    })
+  );
+  const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
+  expect(result.contents.toString()).toEqual('heythere');
+  await tools.delayFor(1100);
+  const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey');
+  expect(result2).toBeNull();
 });
 
 tap.start();

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@push.rocks/levelcache',
+  version: '3.0.5',
+  description: 'a cache that uses memory/disk/s3 as backup'
+}

ts/index.ts

@@ -1,2 +1,2 @@
-export * from './levelcache.classes.levelcache';
-export * from './levelcache.classes.cacheentry';
+export * from './levelcache.classes.levelcache.js';
+export * from './levelcache.classes.cacheentry.js';

ts/levelcache.abstract.classes.cache.ts

@@ -1,25 +1,38 @@
-import { CacheEntry } from './levelcache.classes.cacheentry';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
 
 export abstract class AbstractCache {
   public abstract ready: Promise<void>;
   public abstract status: 'active' | 'inactive';
 
-  // Blobs
+  // Cache Entries
   /**
    * store a Blob
    */
   public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
 
-  // Cache Entries
   /**
    * retrieve cache entry
    */
   public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
 
+  /**
+   * checks for the presence of a key
+   * @param keyArg
+   */
   public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
 
   /**
-   * cleans the cache
+   * delete a key
    */
-  public abstract clean(): Promise<void>;
+  public abstract deleteCacheEntryByKey(keyArg: string): Promise<void>;
+
+  /**
+   * clean the cache
+   */
+  public abstract cleanOutdated(): Promise<void>;
+
+  /**
+   * cleans the complete cache
+   */
+  public abstract cleanAll(): Promise<void>;
 }

ts/levelcache.classes.cache.diskmanager.ts

@@ -1,8 +1,8 @@
-import * as plugins from './levelcache.plugins';
-import * as paths from './levelcache.paths';
-import { AbstractCache } from './levelcache.abstract.classes.cache';
-import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
-import { CacheEntry } from './levelcache.classes.cacheentry';
+import * as plugins from './levelcache.plugins.js';
+import * as paths from './levelcache.paths.js';
+import { AbstractCache } from './levelcache.abstract.classes.cache.js';
+import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
 
 /**
  *
@@ -51,14 +51,17 @@ export class CacheDiskManager extends AbstractCache {
     );
   }
 
-  public async checkKeyPresence(keyArg): Promise<boolean> {
+  public async checkKeyPresence(keyArg: string): Promise<boolean> {
     return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
   }
 
-  /**
-   * cleans the DiskCache directory
-   */
-  public async clean() {
+  public async deleteCacheEntryByKey(keyArg: string) {
+    await plugins.smartfile.fs.remove(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
+  }
+
+  public async cleanOutdated() {}
+
+  public async cleanAll() {
     if (this.status === 'active') {
       if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
         await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);

ts/levelcache.classes.cache.memorymanager.ts

@@ -1,7 +1,7 @@
-import * as plugins from './levelcache.plugins';
-import { AbstractCache } from './levelcache.abstract.classes.cache';
-import { CacheEntry } from './levelcache.classes.cacheentry';
-import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
+import * as plugins from './levelcache.plugins.js';
+import { AbstractCache } from './levelcache.abstract.classes.cache.js';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
+import { type ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache.js';
 
 export class CacheMemoryManager extends AbstractCache {
   private levelCacheRef: LevelCache;
@@ -38,7 +38,13 @@ export class CacheMemoryManager extends AbstractCache {
     }
   }
 
-  public async clean() {
+  public async deleteCacheEntryByKey(keyArg: string) {
+    this.fastMap.removeFromMap(keyArg);
+  }
+
+  public async cleanOutdated() {}
+
+  public async cleanAll() {
     this.fastMap.clean();
   }
 }

ts/levelcache.classes.cache.s3manager.ts

@@ -1,7 +1,7 @@
-import * as plugins from './levelcache.plugins';
-import { AbstractCache } from './levelcache.abstract.classes.cache';
-import { LevelCache } from './levelcache.classes.levelcache';
-import { CacheEntry } from './levelcache.classes.cacheentry';
+import * as plugins from './levelcache.plugins.js';
+import { AbstractCache } from './levelcache.abstract.classes.cache.js';
+import { LevelCache } from './levelcache.classes.levelcache.js';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
 
 /**
  *
@@ -26,9 +26,9 @@ export class CacheS3Manager extends AbstractCache {
     if (this.levelCacheRef.options.s3Config) {
       this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
       this.s3CacheBucket = await this.smartbucket.getBucketByName('');
-      this.s3CacheDir = await (await this.s3CacheBucket.getBaseDirectory()).getSubDirectoryByName(
-        this.levelCacheRef.options.cacheId
-      );
+      this.s3CacheDir = await (
+        await this.s3CacheBucket.getBaseDirectory()
+      ).getSubDirectoryByName(this.levelCacheRef.options.cacheId);
       if (this.levelCacheRef.options.maxS3StorageInMB) {
         console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
       } else {
@@ -64,7 +64,18 @@ export class CacheS3Manager extends AbstractCache {
     return false;
   }
 
-  public async clean() {
+  public async deleteCacheEntryByKey(keyArg: string) {
+    if (this.status === 'active') {
+      await this.s3CacheDir.fastRemove(encodeURIComponent(keyArg));
+    }
+  }
+
+  /**
+   * clean outdated
+   */
+  public async cleanOutdated() {}
+
+  public async cleanAll() {
     await this.s3CacheDir.deleteWithAllContents();
   }
 }

ts/levelcache.classes.cacheentry.ts

@@ -1,4 +1,4 @@
-import * as plugins from './levelcache.plugins';
+import * as plugins from './levelcache.plugins.js';
 
 export interface ICacheEntryConstructorOptions {
   key?: string;
@@ -12,7 +12,8 @@ export interface ICacheEntryConstructorOptions {
  */
 export class CacheEntry
   extends plugins.smartjson.Smartjson
-  implements ICacheEntryConstructorOptions {
+  implements ICacheEntryConstructorOptions
+{
   public static fromStorageJsonString(storageJsonString: string) {
     return new CacheEntry(plugins.smartjson.parse(storageJsonString));
   }
@@ -23,6 +24,9 @@ export class CacheEntry
   @plugins.smartjson.foldDec()
   public ttl: number;
 
+  @plugins.smartjson.foldDec()
+  public createdAt: number;
+
   @plugins.smartjson.foldDec()
   public typeInfo: string;
 

ts/levelcache.classes.cacherouter.ts

@@ -1,7 +1,7 @@
-import * as plugins from './levelcache.plugins';
-import { LevelCache } from './levelcache.classes.levelcache';
-import { AbstractCache } from './levelcache.abstract.classes.cache';
-import { CacheEntry } from './levelcache.classes.cacheentry';
+import * as plugins from './levelcache.plugins.js';
+import { LevelCache } from './levelcache.classes.levelcache.js';
+import { AbstractCache } from './levelcache.abstract.classes.cache.js';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
 
 export class CacheRouter {
   public levelCacheRef: LevelCache;
@@ -14,24 +14,31 @@ export class CacheRouter {
   /**
    * gets the relevant cache to perform a store action on
    */
-  async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
+  async getCacheForStoreAction (keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
     let returnCache: AbstractCache;
+    const mbToBytesMultiplier = 1000 * 1000;
+    const maxMemoryBytes = this.levelCacheRef.options.maxMemoryStorageInMB * mbToBytesMultiplier;
+    const maxDiskBytes = this.levelCacheRef.options.maxDiskStorageInMB * mbToBytesMultiplier;
+    const maxS3Bytes = this.levelCacheRef.options.maxS3StorageInMB * mbToBytesMultiplier;
+
     switch (true) {
-      case cacheEntry.contents.byteLength <= 500:
+      case cacheEntry.contents.byteLength <= maxMemoryBytes &&
+        this.levelCacheRef.cacheMemoryManager.status === 'active':
         returnCache = this.levelCacheRef.cacheMemoryManager;
         break;
       case this.levelCacheRef.cacheDiskManager.status === 'active' &&
-        cacheEntry.contents.byteLength >= 500 &&
-        (cacheEntry.contents.byteLength < 10000 ||
-          this.levelCacheRef.cacheS3Manager.status === 'inactive'):
+        cacheEntry.contents.byteLength > maxMemoryBytes &&
+        cacheEntry.contents.byteLength <= maxDiskBytes &&
+        this.levelCacheRef.cacheDiskManager.status === 'active':
         returnCache = this.levelCacheRef.cacheDiskManager;
         break;
-      case cacheEntry.contents.byteLength >= 10000 &&
+      case cacheEntry.contents.byteLength > maxDiskBytes &&
+        cacheEntry.contents.byteLength < maxS3Bytes &&
         this.levelCacheRef.cacheS3Manager.status === 'active':
         returnCache = this.levelCacheRef.cacheS3Manager;
         break;
       default:
-        returnCache = this.levelCacheRef.cacheMemoryManager;
+        returnCache = null;
     }
     this.cacheKeyMap.addToMap(keyArg, returnCache);
     return returnCache;
@@ -40,7 +47,7 @@ export class CacheRouter {
   /**
    * gets the relevant cache to perform a retrieval action on
    */
-  async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
+  async getCacheForRetrieveAction (keyArg: string): Promise<AbstractCache> {
     const done = plugins.smartpromise.defer<AbstractCache>();
     const returnCache = this.cacheKeyMap.getByKey(keyArg);
     if (!returnCache && this.levelCacheRef.options.persistentCache) {

ts/levelcache.classes.levelcache.ts

@@ -1,10 +1,10 @@
-import * as plugins from './levelcache.plugins';
-import { CacheDiskManager } from './levelcache.classes.cache.diskmanager';
-import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
-import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
-import { CacheEntry } from './levelcache.classes.cacheentry';
-import { CacheRouter } from './levelcache.classes.cacherouter';
-import { AbstractCache } from './levelcache.abstract.classes.cache';
+import * as plugins from './levelcache.plugins.js';
+import { CacheDiskManager } from './levelcache.classes.cache.diskmanager.js';
+import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager.js';
+import { CacheS3Manager } from './levelcache.classes.cache.s3manager.js';
+import { CacheEntry } from './levelcache.classes.cacheentry.js';
+import { CacheRouter } from './levelcache.classes.cacherouter.js';
+import { AbstractCache } from './levelcache.abstract.classes.cache.js';
 
 export interface ILevelCacheConstructorOptions {
   /**
@@ -42,13 +42,18 @@ export class LevelCache extends AbstractCache {
 
   constructor(optionsArg: ILevelCacheConstructorOptions) {
     super();
-    this.options = optionsArg;
+    this.options = {
+      maxMemoryStorageInMB: 0.5,
+      maxDiskStorageInMB: 10,
+      maxS3StorageInMB: 50,
+      ...optionsArg
+    };
     this.init();
   }
 
   public async init() {
-    this.cacheDiskManager = new CacheDiskManager(this);
     this.cacheMemoryManager = new CacheMemoryManager(this);
+    this.cacheDiskManager = new CacheDiskManager(this);
     this.cacheS3Manager = new CacheS3Manager(this);
     await Promise.all([
       this.cacheDiskManager.ready,
@@ -62,6 +67,7 @@ export class LevelCache extends AbstractCache {
   public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
     cacheEntryArg.key = keyArg;
     const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
+    cacheEntryArg.createdAt = Date.now();
     await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
   }
 
@@ -73,6 +79,10 @@ export class LevelCache extends AbstractCache {
     const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
     if (targetCache) {
       const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
+      if (cacheEntry.createdAt + cacheEntry.ttl < Date.now()) {
+        await this.deleteCacheEntryByKey(keyArg).catch();
+        return null;
+      }
       return cacheEntry;
     } else {
       return null;
@@ -87,15 +97,25 @@ export class LevelCache extends AbstractCache {
     ]);
   }
 
-  // cache maintenance
-  /**
-   * cleans the cache
-   */
-  public async clean(): Promise<void> {
+  public async deleteCacheEntryByKey(keyArg) {
     await Promise.all([
-      this.cacheDiskManager.clean(),
-      this.cacheDiskManager.clean(),
-      this.cacheS3Manager.clean(),
+      this.cacheMemoryManager.deleteCacheEntryByKey(keyArg),
+      this.cacheDiskManager.deleteCacheEntryByKey(keyArg),
+      this.cacheS3Manager.deleteCacheEntryByKey(keyArg),
+    ]);
+  }
+
+  // cache maintenance
+  public async cleanOutdated() {}
+
+  /**
+   * cleans the complete cache
+   */
+  public async cleanAll(): Promise<void> {
+    await Promise.all([
+      this.cacheDiskManager.cleanAll(),
+      this.cacheDiskManager.cleanAll(),
+      this.cacheS3Manager.cleanAll(),
     ]);
   }
 }

ts/levelcache.paths.ts

@@ -1,4 +1,7 @@
-import * as plugins from './levelcache.plugins';
+import * as plugins from './levelcache.plugins.js';
 
-export const packageDir = plugins.path.join(__dirname, '../');
+export const packageDir = plugins.path.join(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../'
+);
 export const nogitDir = plugins.path.join(packageDir, '.nogit/');

ts/levelcache.plugins.ts

@@ -3,16 +3,17 @@ import * as path from 'path';
 
 export { path };
 
-// @pushrocks scope
-import * as lik from '@pushrocks/lik';
-import * as smartbucket from '@pushrocks/smartbucket';
-import * as smartcache from '@pushrocks/smartcache';
-import * as smartfile from '@pushrocks/smartfile';
-import * as smartjson from '@pushrocks/smartjson';
-import * as smartpromise from '@pushrocks/smartpromise';
-import * as smartstring from '@pushrocks/smartstring';
-import * as smartunique from '@pushrocks/smartunique';
-import * as taskbuffer from '@pushrocks/taskbuffer';
+// @push.rocks scope
+import * as lik from '@push.rocks/lik';
+import * as smartbucket from '@push.rocks/smartbucket';
+import * as smartcache from '@push.rocks/smartcache';
+import * as smartfile from '@push.rocks/smartfile';
+import * as smartjson from '@push.rocks/smartjson';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartstring from '@push.rocks/smartstring';
+import * as smartunique from '@push.rocks/smartunique';
+import * as taskbuffer from '@push.rocks/taskbuffer';
 
 export {
   lik,
@@ -20,6 +21,7 @@ export {
   smartcache,
   smartfile,
   smartjson,
+  smartpath,
   smartpromise,
   smartstring,
   smartunique,

tsconfig.json

@@ -1,7 +1,11 @@
 {
   "compilerOptions": {
     "experimentalDecorators": true,
-    "target": "es2017",
-    "module": "commonjs"
+    "useDefineForClassFields": false,
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "nodenext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true,
   }
-}
+}

tslint.json

@@ -1,17 +0,0 @@
-{
-  "extends": ["tslint:latest", "tslint-config-prettier"],
-  "rules": {
-    "semicolon": [true, "always"],
-    "no-console": false,
-    "ordered-imports": false,
-    "object-literal-sort-keys": false,
-    "member-ordering": {
-      "options":{
-        "order": [
-          "static-method"
-        ]
-      }
-    }
-  },
-  "defaultSeverity": "warning"
-}