fix(core): update

package.json

@@ -3,30 +3,33 @@
   "version": "1.0.6",
   "private": false,
   "description": "a cache that uses memory/disk/s3 as backup",
-  "main": "dist/index.js",
-  "typings": "dist/index.d.ts",
+  "main": "dist_ts/index.js",
+  "typings": "dist_ts/index.d.ts",
   "author": "Lossless GmbH",
   "license": "MIT",
   "scripts": {
-    "test": "(tstest test/)",
-    "build": "(tsbuild)",
-    "format": "(gitzone format)"
+    "test": "(tstest test/ --web)",
+    "build": "(tsbuild --web)"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.0.22",
-    "@gitzone/tstest": "^1.0.15",
-    "@pushrocks/tapbundle": "^3.0.7",
-    "@types/node": "^13.7.1",
-    "tslint": "^6.0.0",
+    "@gitzone/tsbuild": "^2.1.25",
+    "@gitzone/tstest": "^1.0.52",
+    "@pushrocks/tapbundle": "^3.2.14",
+    "@types/node": "^14.14.41",
+    "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.15.0"
   },
   "dependencies": {
-    "@pushrocks/lik": "^3.0.18",
-    "@pushrocks/smartbucket": "^1.0.24",
+    "@pushrocks/lik": "^4.0.20",
+    "@pushrocks/smartbucket": "^1.0.39",
     "@pushrocks/smartcache": "^1.0.13",
-    "@pushrocks/smartfile": "^7.0.8",
-    "@pushrocks/smartstring": "^3.0.18",
-    "@pushrocks/smartunique": "^3.0.1"
+    "@pushrocks/smartexit": "^1.0.15",
+    "@pushrocks/smartfile": "^8.0.9",
+    "@pushrocks/smartjson": "^4.0.6",
+    "@pushrocks/smartpromise": "^3.1.5",
+    "@pushrocks/smartstring": "^3.0.24",
+    "@pushrocks/smartunique": "^3.0.3",
+    "@pushrocks/taskbuffer": "^2.1.13"
   },
   "files": [
     "ts/**/*",

readme.md

@@ -18,6 +18,9 @@ a cache that uses memory/disk/s3 as backup
 
 ## Usage
 
+@pushrocks/levelcache implements an efficient way to cache larger amounts of data ind a multilevel approach.
+smaller and prioritized data is safed in memory.
+
 Use TypeScript for best in class intellisense.
 
 ## Contribution

test/test.ts

@@ -1,17 +1,24 @@
 import { expect, tap } from '@pushrocks/tapbundle';
 import * as levelcache from '../ts/index';
+import { CacheEntry } from '../ts/index';
 
 let testLevelCache: levelcache.LevelCache;
 
 tap.test('should create a new levelcache instance', async () => {
   testLevelCache = new levelcache.LevelCache({
-    
+    cacheId: 'myCache'
   });
   expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
 });
 
 tap.test('should cache a value', async () => {
-  testLevelCache.
+  await testLevelCache.storeCacheEntryByKey('mykey', new CacheEntry({
+    contents: Buffer.from('heythere'),
+    ttl: 10000,
+    typeInfo: 'string'
+  }));
+  const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
+  console.log(result);
 });
 
 tap.start();

ts/index.ts

@@ -1 +1,3 @@
 export * from './levelcache.classes.levelcache';
+export * from './levelcache.classes.cacheentry';
+

ts/levelcache.abstract.classes.cache.ts

@@ -0,0 +1,25 @@
+import { CacheEntry } from "./levelcache.classes.cacheentry";
+
+export abstract class AbstractCache {
+  public abstract ready: Promise<void>;
+  public abstract status: 'active' | 'inactive';
+  
+  // Blobs
+  /**
+   * store a Blob
+   */
+  public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
+
+  // Cache Entries
+  /**
+   * retrieve cache entry
+   */
+  public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
+
+  public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
+
+  /**
+   * cleans the cache
+   */
+  public abstract clean(): Promise<void>;
+}

ts/levelcache.classes.cache.diskmanager.ts

@@ -1,6 +1,60 @@
 import * as plugins from './levelcache.plugins';
+import * as paths from './levelcache.paths';
+import { AbstractCache } from './levelcache.abstract.classes.cache';
+import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
+import { CacheEntry } from './levelcache.classes.cacheentry';
 
 /**
  *
  */
-export class CacheDiskManager {}
+export class CacheDiskManager extends AbstractCache {
+  private levelCacheRef: LevelCache;
+  private readyDeferred = plugins.smartpromise.defer<void>();
+  
+  public ready = this.readyDeferred.promise;
+  public status: 'active' | 'inactive';
+  public fsPath: string;
+  public maxCacheSizeInMb: number;
+
+  constructor(levelCacheRefArg: LevelCache) {
+    super();
+    this.levelCacheRef = levelCacheRefArg;
+    this.init();
+  }
+
+  public async init() {
+    if (this.levelCacheRef.options.diskStoragePath) {
+      this.fsPath = plugins.path.join(this.levelCacheRef.options.diskStoragePath, this.levelCacheRef.options.cacheId);
+    } else {
+      this.fsPath = plugins.path.join(paths.nogitDir, this.levelCacheRef.options.cacheId);
+    }
+    if (this.status === 'active') {
+      plugins.smartfile.fs.ensureDirSync(this.fsPath);
+    }
+    this.readyDeferred.resolve();
+  }
+
+  public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
+    const fileString = await plugins.smartfile.fs.toStringSync(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
+    return CacheEntry.fromStorageJsonString(fileString);
+  }
+
+  public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
+    await plugins.smartfile.memory.toFs(cacheEntryArg.foldToJson(), plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
+  }
+
+  public async checkKeyPresence(keyArg): Promise<boolean> {
+    return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
+  }
+
+  /**
+   * cleans the DiskCache directory
+   */
+  public async clean() {
+    if (this.status === 'active') {
+      if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
+        await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);
+      }
+    }
+  }
+}

ts/levelcache.classes.cache.memorymanager.ts

@@ -1,3 +1,45 @@
 import * as plugins from './levelcache.plugins';
+import { AbstractCache } from './levelcache.abstract.classes.cache';
+import { CacheEntry } from './levelcache.classes.cacheentry';
+import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
 
-export class CacheMemoryManager {}
+export class CacheMemoryManager extends AbstractCache {
+  private levelCacheRef: LevelCache;
+  private fastMap = new plugins.lik.FastMap<CacheEntry>();
+  private readyDeferred = plugins.smartpromise.defer<void>();
+  
+  public ready = this.readyDeferred.promise;
+  public status: 'active' | 'inactive';
+
+  constructor(levelCacheRefArg: LevelCache) {
+    super();
+    this.levelCacheRef = levelCacheRefArg;
+    this.init();
+  }
+
+  public async init() {
+    this.status = 'active';
+    this.readyDeferred.resolve();
+  }
+
+  public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
+    this.fastMap.addToMap(keyArg, cacheEntryArg, { force: true });
+  }
+
+  public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
+    return this.fastMap.getByKey(keyArg);
+  }
+
+  public async checkKeyPresence(keyArg: string): Promise<boolean> {
+    if (this.fastMap.getByKey(keyArg)) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  public async clean() {
+    this.fastMap.clean();
+  }
+  
+}

ts/levelcache.classes.cache.s3manager.ts

@@ -1,6 +1,67 @@
 import * as plugins from './levelcache.plugins';
+import { AbstractCache } from './levelcache.abstract.classes.cache';
+import { LevelCache } from './levelcache.classes.levelcache';
+import { CacheEntry } from './levelcache.classes.cacheentry';
 
 /**
  *
  */
-export class CacheS3Manager {}
+export class CacheS3Manager extends AbstractCache {
+  private levelCacheRef: LevelCache;
+  private smartbucket: plugins.smartbucket.SmartBucket;
+  private s3CacheBucket: plugins.smartbucket.Bucket;
+  private s3CacheDir: plugins.smartbucket.Directory;
+  private readyDeferred = plugins.smartpromise.defer<void>();
+
+  public ready = this.readyDeferred.promise;
+  public status: 'active' | 'inactive';
+
+  constructor(levelCacheRefArg: LevelCache) {
+    super();
+    this.levelCacheRef = levelCacheRefArg;
+    this.init();
+  }
+
+  public async init() {
+    if (this.levelCacheRef.options.s3Config) {
+      this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
+      this.s3CacheBucket = await this.smartbucket.getBucketByName('');
+      this.s3CacheDir = await (await this.s3CacheBucket.getBaseDirectory()).getSubDirectoryByName(
+        this.levelCacheRef.options.cacheId
+      );
+      if (this.levelCacheRef.options.maxS3StorageInMB) {
+        console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
+      } else {
+        console.log(`s3 cache started without limit. Automatically applying timebox of 1 month`);
+      }
+      this.status = 'active';
+    } else {
+      this.status = 'inactive';
+    }
+    this.readyDeferred.resolve();
+  }
+
+  public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
+    const jsonFileString = (await this.s3CacheDir.fastGet(encodeURIComponent(keyArg))).toString();
+    const cacheEntry = CacheEntry.fromStorageJsonString(jsonFileString);
+    return cacheEntry;
+  }
+
+  public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
+    await this.s3CacheDir.fastStore(encodeURIComponent(keyArg), cacheEntryArg.toStorageJsonString());
+  }
+
+  public async checkKeyPresence(keyArg: string): Promise<boolean> {
+    const files = await this.s3CacheDir.listFiles();
+    for (const file of files) {
+      if (file.name === keyArg) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public async clean() {
+    await this.s3CacheDir.deleteWithAllContents();
+  }
+}

ts/levelcache.classes.cacheentry.ts

@@ -1,20 +1,38 @@
 import * as plugins from './levelcache.plugins';
 
-export interface ICacheEntryConstructorOptions {}
+export interface ICacheEntryConstructorOptions {
+  key?: string;
+  ttl: number;
+  typeInfo?: string;
+  contents: Buffer;
+}
 
 /**
  * a CacheEntry
  */
-export class CacheEntry {
-  public static fromBuffer() {}
-  public static fromString() {}
-
-  type: 'string' | 'blob';
-  mode: 'complete' | 'stream';
-  keyArg: string;
-  cacheStream: ReadableStream;
-  cacheContents: Buffer;
-  cacheValue: string;
-
-  constructor(optionsArg: {}) {}
+export class CacheEntry extends plugins.smartjson.Smartjson implements ICacheEntryConstructorOptions {
+  public static fromStorageJsonString(storageJsonString: string) {
+    return new CacheEntry(plugins.smartjson.parse(storageJsonString));
+  }
+
+  @plugins.smartjson.foldDec()
+  public key: string;
+
+  @plugins.smartjson.foldDec()
+  public ttl: number;
+
+  @plugins.smartjson.foldDec()
+  public typeInfo: string;
+  
+  @plugins.smartjson.foldDec()
+  contents: Buffer;
+
+  public toStorageJsonString(): string {
+    return this.foldToJson();
+  }
+
+  constructor(optionsArg: ICacheEntryConstructorOptions) {
+    super();
+    Object.assign(this, optionsArg);
+  }
 }

ts/levelcache.classes.cacherouter.ts

@@ -1,9 +1,11 @@
 import * as plugins from './levelcache.plugins';
 import { LevelCache } from './levelcache.classes.levelcache';
+import { AbstractCache } from './levelcache.abstract.classes.cache';
+import { CacheEntry } from './levelcache.classes.cacheentry';
 
 export class CacheRouter {
   public levelCacheRef: LevelCache;
-  public cacheMap;
+  public cacheKeyMap = new plugins.lik.FastMap<AbstractCache>();
 
   constructor(levelCacheRef: LevelCache) {
     this.levelCacheRef = levelCacheRef;
@@ -12,10 +14,55 @@ export class CacheRouter {
   /**
    * gets the relevant cache to perform a store action on
    */
-  async getCacheForStoreAction() {}
+  async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
+    let returnCache: AbstractCache;
+    switch (true) {
+      case cacheEntry.contents.byteLength <= 500:
+        returnCache = this.levelCacheRef.cacheMemoryManager;
+        break;
+      case this.levelCacheRef.cacheDiskManager.status === 'active' &&
+        cacheEntry.contents.byteLength >= 500 &&
+        (cacheEntry.contents.byteLength < 10000 ||
+          this.levelCacheRef.cacheS3Manager.status === 'inactive'):
+        returnCache = this.levelCacheRef.cacheDiskManager;
+        break;
+      case cacheEntry.contents.byteLength >= 10000 &&
+        this.levelCacheRef.cacheS3Manager.status === 'active':
+        returnCache = this.levelCacheRef.cacheS3Manager;
+        break;
+      default:
+        returnCache = this.levelCacheRef.cacheMemoryManager;
+    }
+    this.cacheKeyMap.addToMap(keyArg, returnCache);
+    return returnCache;
+  }
 
   /**
    * gets the relevant cache to perform a retrieval action on
    */
-  async getCacheForRetrieveAction() {}
+  async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
+    const done = plugins.smartpromise.defer<AbstractCache>();
+    const returnCache = this.cacheKeyMap.getByKey(keyArg);
+    if (!returnCache && this.levelCacheRef.options.persistentCache) {
+      const checkCache = (cacheArg: AbstractCache) => {
+        const resultPromise = cacheArg.checkKeyPresence(keyArg);
+        resultPromise.then(hasKeyArg => {
+          if (hasKeyArg) {
+            done.resolve(cacheArg);
+          }
+        });
+        return resultPromise;
+      };
+      Promise.all([
+        checkCache(this.levelCacheRef.cacheMemoryManager),
+        checkCache(this.levelCacheRef.cacheDiskManager),
+        checkCache(this.levelCacheRef.cacheMemoryManager)
+      ]).then(() => {
+        done.resolve(returnCache);
+      });
+    } else {
+      done.resolve(returnCache);
+    }
+    return done.promise;
+  }
 }

ts/levelcache.classes.levelcache.ts

@@ -4,57 +4,99 @@ import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
 import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
 import { CacheEntry } from './levelcache.classes.cacheentry';
 import { CacheRouter } from './levelcache.classes.cacherouter';
+import { AbstractCache } from './levelcache.abstract.classes.cache';
 
 export interface ILevelCacheConstructorOptions {
-  maxMemoryStorageInMB: number;
-  maxDiskStorageInMB: number;
+  /**
+   * a unique id when having more than one cache
+   */
+  cacheId: string;
+  maxMemoryStorageInMB?: number;
+  maxDiskStorageInMB?: number;
   maxS3StorageInMB?: number;
-  smartbucketConfig?: plugins.smartbucket.ISmartBucketConfig;
+  diskStoragePath?: string;
+  s3Config?: plugins.smartbucket.ISmartBucketConfig;
+  s3BucketName?: string;
+  forceLevel?: 'memory' | 'disk' | 's3';
+  expirationInMs?: number;
+  immutableCache?: boolean;
+  persistentCache?: boolean;
 }
 
 /**
  * a leveled cache for storing things for a short time
  */
-export class LevelCache {
+export class LevelCache extends AbstractCache {
+  private readyDeferred = plugins.smartpromise.defer<void>();
+
+  public ready = this.readyDeferred.promise;
+
+  public status: 'active' = 'active'; // artifact of AbstractCache
+
   public cacheRouter = new CacheRouter(this);
-  public cacheDiskManager = new CacheDiskManager();
-  public cacheMemoryManager = new CacheMemoryManager();
-  public cacheS3Manager = new CacheS3Manager();
+  public cacheDiskManager: CacheDiskManager;
+  public cacheMemoryManager: CacheMemoryManager;
+  public cacheS3Manager: CacheS3Manager;
 
   public options: ILevelCacheConstructorOptions;
 
   constructor(optionsArg: ILevelCacheConstructorOptions) {
+    super();
     this.options = optionsArg;
+    this.init();
   }
 
-  private processKey(keyArg: string) {
-    if (!keyArg) {
-      return plugins.smartunique.shortId();
-    }
+  public async init() {
+    this.cacheDiskManager = new CacheDiskManager(this);
+    this.cacheMemoryManager = new CacheMemoryManager(this);
+    this.cacheS3Manager = new CacheS3Manager(this);
+    await Promise.all([
+      this.cacheDiskManager.ready,
+      this.cacheMemoryManager.ready,
+      this.cacheS3Manager.ready,
+    ]);
+    this.readyDeferred.resolve();
   }
 
-  // Blobs
-  /**
-   * store a Blob
-   */
-  public async storeBlobByKey(keyArg: string, blob: Buffer) {
-    keyArg = this.processKey(keyArg);
-    return keyArg;
+  // store things
+  public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
+    cacheEntryArg.key = keyArg;
+    const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
+    await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
   }
 
-  /**
-   * retrieve a blob
-   */
-  public async retrieveBlob(keyArg: string): CacheEntry {}
-
-  // Cache Entries
+  // retrieve things
   /**
    * retrieve cache entry
    */
-  public async retrieveCacheEntryByKey(): CacheEntry {}
+  public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
+    const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
+    if (targetCache) {
+      const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
+      return cacheEntry;
+    } else {
+      return null;
+    }
+  }
 
+
+  public async checkKeyPresence(keyArg: string): Promise<boolean> {
+    return plugins.smartpromise.getFirstTrueOrFalse([
+      this.cacheMemoryManager.checkKeyPresence(keyArg),
+      this.cacheDiskManager.checkKeyPresence(keyArg),
+      this.cacheS3Manager.checkKeyPresence(keyArg)
+    ]);
+  }
+
+  // cache maintenance
   /**
    * cleans the cache
    */
-  public clean() {}
+  public async clean(): Promise<void> {
+    await Promise.all([
+      this.cacheDiskManager.clean(),
+      this.cacheDiskManager.clean(),
+      this.cacheS3Manager.clean(),
+    ]);
+  }
 }

ts/levelcache.paths.ts

@@ -0,0 +1,4 @@
+import * as plugins from './levelcache.plugins';
+
+export const packageDir = plugins.path.join(__dirname, '../');
+export const nogitDir = plugins.path.join(packageDir, '.nogit/');

ts/levelcache.plugins.ts

@@ -8,7 +8,20 @@ import * as lik from '@pushrocks/lik';
 import * as smartbucket from '@pushrocks/smartbucket';
 import * as smartcache from '@pushrocks/smartcache';
 import * as smartfile from '@pushrocks/smartfile';
+import * as smartjson from '@pushrocks/smartjson';
+import * as smartpromise from '@pushrocks/smartpromise';
 import * as smartstring from '@pushrocks/smartstring';
 import * as smartunique from '@pushrocks/smartunique';
+import * as taskbuffer from '@pushrocks/taskbuffer';
 
-export { lik, smartbucket, smartcache, smartfile, smartstring, smartunique };
+export {
+  lik,
+  smartbucket,
+  smartcache,
+  smartfile,
+  smartjson,
+  smartpromise,
+  smartstring,
+  smartunique,
+  taskbuffer,
+};

tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "target": "es2017",
+    "module": "commonjs"
+  }
+}