fix(core): update

This commit is contained in:
Philipp Kunz 2021-04-23 18:40:57 +00:00
parent be8227f451
commit e53ebc74f8
15 changed files with 12810 additions and 1177 deletions

13261
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -3,30 +3,33 @@
"version": "1.0.6",
"private": false,
"description": "a cache that uses memory/disk/s3 as backup",
"main": "dist/index.js",
"typings": "dist/index.d.ts",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/)",
"build": "(tsbuild)",
"format": "(gitzone format)"
"test": "(tstest test/ --web)",
"build": "(tsbuild --web)"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.0.22",
"@gitzone/tstest": "^1.0.15",
"@pushrocks/tapbundle": "^3.0.7",
"@types/node": "^13.7.1",
"tslint": "^6.0.0",
"@gitzone/tsbuild": "^2.1.25",
"@gitzone/tstest": "^1.0.52",
"@pushrocks/tapbundle": "^3.2.14",
"@types/node": "^14.14.41",
"tslint": "^6.1.3",
"tslint-config-prettier": "^1.15.0"
},
"dependencies": {
"@pushrocks/lik": "^3.0.18",
"@pushrocks/smartbucket": "^1.0.24",
"@pushrocks/lik": "^4.0.20",
"@pushrocks/smartbucket": "^1.0.39",
"@pushrocks/smartcache": "^1.0.13",
"@pushrocks/smartfile": "^7.0.8",
"@pushrocks/smartstring": "^3.0.18",
"@pushrocks/smartunique": "^3.0.1"
"@pushrocks/smartexit": "^1.0.15",
"@pushrocks/smartfile": "^8.0.9",
"@pushrocks/smartjson": "^4.0.6",
"@pushrocks/smartpromise": "^3.1.5",
"@pushrocks/smartstring": "^3.0.24",
"@pushrocks/smartunique": "^3.0.3",
"@pushrocks/taskbuffer": "^2.1.13"
},
"files": [
"ts/**/*",

View File

@ -18,6 +18,9 @@ a cache that uses memory/disk/s3 as backup
## Usage
@pushrocks/levelcache implements an efficient way to cache larger amounts of data ind a multilevel approach.
smaller and prioritized data is safed in memory.
Use TypeScript for best in class intellisense.
## Contribution

View File

@ -1,17 +1,24 @@
import { expect, tap } from '@pushrocks/tapbundle';
import * as levelcache from '../ts/index';
import { CacheEntry } from '../ts/index';
let testLevelCache: levelcache.LevelCache;
tap.test('should create a new levelcache instance', async () => {
testLevelCache = new levelcache.LevelCache({
cacheId: 'myCache'
});
expect(testLevelCache).to.be.instanceOf(levelcache.LevelCache);
});
tap.test('should cache a value', async () => {
testLevelCache.
await testLevelCache.storeCacheEntryByKey('mykey', new CacheEntry({
contents: Buffer.from('heythere'),
ttl: 10000,
typeInfo: 'string'
}));
const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
console.log(result);
});
tap.start();

View File

@ -1 +1,3 @@
export * from './levelcache.classes.levelcache';
export * from './levelcache.classes.cacheentry';

View File

@ -0,0 +1,25 @@
import { CacheEntry } from "./levelcache.classes.cacheentry";
export abstract class AbstractCache {
public abstract ready: Promise<void>;
public abstract status: 'active' | 'inactive';
// Blobs
/**
* store a Blob
*/
public abstract storeCacheEntryByKey(keyArg: string, valueArg: CacheEntry): Promise<void>;
// Cache Entries
/**
* retrieve cache entry
*/
public abstract retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry>;
public abstract checkKeyPresence(keyArg: string): Promise<boolean>;
/**
* cleans the cache
*/
public abstract clean(): Promise<void>;
}

View File

@ -1,6 +1,60 @@
import * as plugins from './levelcache.plugins';
import * as paths from './levelcache.paths';
import { AbstractCache } from './levelcache.abstract.classes.cache';
import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
import { CacheEntry } from './levelcache.classes.cacheentry';
/**
*
*/
export class CacheDiskManager {}
export class CacheDiskManager extends AbstractCache {
private levelCacheRef: LevelCache;
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
public fsPath: string;
public maxCacheSizeInMb: number;
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.diskStoragePath) {
this.fsPath = plugins.path.join(this.levelCacheRef.options.diskStoragePath, this.levelCacheRef.options.cacheId);
} else {
this.fsPath = plugins.path.join(paths.nogitDir, this.levelCacheRef.options.cacheId);
}
if (this.status === 'active') {
plugins.smartfile.fs.ensureDirSync(this.fsPath);
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const fileString = await plugins.smartfile.fs.toStringSync(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
return CacheEntry.fromStorageJsonString(fileString);
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await plugins.smartfile.memory.toFs(cacheEntryArg.foldToJson(), plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
public async checkKeyPresence(keyArg): Promise<boolean> {
return plugins.smartfile.fs.isFile(plugins.path.join(this.fsPath, encodeURIComponent(keyArg)));
}
/**
* cleans the DiskCache directory
*/
public async clean() {
if (this.status === 'active') {
if (plugins.smartfile.fs.isDirectory(this.fsPath)) {
await plugins.smartfile.fs.ensureEmptyDir(this.fsPath);
}
}
}
}

View File

@ -1,3 +1,45 @@
import * as plugins from './levelcache.plugins';
import { AbstractCache } from './levelcache.abstract.classes.cache';
import { CacheEntry } from './levelcache.classes.cacheentry';
import { ILevelCacheConstructorOptions, LevelCache } from './levelcache.classes.levelcache';
export class CacheMemoryManager {}
export class CacheMemoryManager extends AbstractCache {
private levelCacheRef: LevelCache;
private fastMap = new plugins.lik.FastMap<CacheEntry>();
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
this.status = 'active';
this.readyDeferred.resolve();
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
this.fastMap.addToMap(keyArg, cacheEntryArg, { force: true });
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
return this.fastMap.getByKey(keyArg);
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
if (this.fastMap.getByKey(keyArg)) {
return true;
} else {
return false;
}
}
public async clean() {
this.fastMap.clean();
}
}

View File

@ -1,6 +1,67 @@
import * as plugins from './levelcache.plugins';
import { AbstractCache } from './levelcache.abstract.classes.cache';
import { LevelCache } from './levelcache.classes.levelcache';
import { CacheEntry } from './levelcache.classes.cacheentry';
/**
*
*/
export class CacheS3Manager {}
export class CacheS3Manager extends AbstractCache {
private levelCacheRef: LevelCache;
private smartbucket: plugins.smartbucket.SmartBucket;
private s3CacheBucket: plugins.smartbucket.Bucket;
private s3CacheDir: plugins.smartbucket.Directory;
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' | 'inactive';
constructor(levelCacheRefArg: LevelCache) {
super();
this.levelCacheRef = levelCacheRefArg;
this.init();
}
public async init() {
if (this.levelCacheRef.options.s3Config) {
this.smartbucket = new plugins.smartbucket.SmartBucket(this.levelCacheRef.options.s3Config);
this.s3CacheBucket = await this.smartbucket.getBucketByName('');
this.s3CacheDir = await (await this.s3CacheBucket.getBaseDirectory()).getSubDirectoryByName(
this.levelCacheRef.options.cacheId
);
if (this.levelCacheRef.options.maxS3StorageInMB) {
console.log(`cache level S3 activated with ${this.levelCacheRef.options.maxS3StorageInMB}`);
} else {
console.log(`s3 cache started without limit. Automatically applying timebox of 1 month`);
}
this.status = 'active';
} else {
this.status = 'inactive';
}
this.readyDeferred.resolve();
}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const jsonFileString = (await this.s3CacheDir.fastGet(encodeURIComponent(keyArg))).toString();
const cacheEntry = CacheEntry.fromStorageJsonString(jsonFileString);
return cacheEntry;
}
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry) {
await this.s3CacheDir.fastStore(encodeURIComponent(keyArg), cacheEntryArg.toStorageJsonString());
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
const files = await this.s3CacheDir.listFiles();
for (const file of files) {
if (file.name === keyArg) {
return true;
}
}
return false;
}
public async clean() {
await this.s3CacheDir.deleteWithAllContents();
}
}

View File

@ -1,20 +1,38 @@
import * as plugins from './levelcache.plugins';
export interface ICacheEntryConstructorOptions {}
export interface ICacheEntryConstructorOptions {
key?: string;
ttl: number;
typeInfo?: string;
contents: Buffer;
}
/**
* a CacheEntry
*/
export class CacheEntry {
public static fromBuffer() {}
public static fromString() {}
type: 'string' | 'blob';
mode: 'complete' | 'stream';
keyArg: string;
cacheStream: ReadableStream;
cacheContents: Buffer;
cacheValue: string;
constructor(optionsArg: {}) {}
export class CacheEntry extends plugins.smartjson.Smartjson implements ICacheEntryConstructorOptions {
public static fromStorageJsonString(storageJsonString: string) {
return new CacheEntry(plugins.smartjson.parse(storageJsonString));
}
@plugins.smartjson.foldDec()
public key: string;
@plugins.smartjson.foldDec()
public ttl: number;
@plugins.smartjson.foldDec()
public typeInfo: string;
@plugins.smartjson.foldDec()
contents: Buffer;
public toStorageJsonString(): string {
return this.foldToJson();
}
constructor(optionsArg: ICacheEntryConstructorOptions) {
super();
Object.assign(this, optionsArg);
}
}

View File

@ -1,9 +1,11 @@
import * as plugins from './levelcache.plugins';
import { LevelCache } from './levelcache.classes.levelcache';
import { AbstractCache } from './levelcache.abstract.classes.cache';
import { CacheEntry } from './levelcache.classes.cacheentry';
export class CacheRouter {
public levelCacheRef: LevelCache;
public cacheMap;
public cacheKeyMap = new plugins.lik.FastMap<AbstractCache>();
constructor(levelCacheRef: LevelCache) {
this.levelCacheRef = levelCacheRef;
@ -12,10 +14,55 @@ export class CacheRouter {
/**
* gets the relevant cache to perform a store action on
*/
async getCacheForStoreAction() {}
async getCacheForStoreAction(keyArg: string, cacheEntry: CacheEntry): Promise<AbstractCache> {
let returnCache: AbstractCache;
switch (true) {
case cacheEntry.contents.byteLength <= 500:
returnCache = this.levelCacheRef.cacheMemoryManager;
break;
case this.levelCacheRef.cacheDiskManager.status === 'active' &&
cacheEntry.contents.byteLength >= 500 &&
(cacheEntry.contents.byteLength < 10000 ||
this.levelCacheRef.cacheS3Manager.status === 'inactive'):
returnCache = this.levelCacheRef.cacheDiskManager;
break;
case cacheEntry.contents.byteLength >= 10000 &&
this.levelCacheRef.cacheS3Manager.status === 'active':
returnCache = this.levelCacheRef.cacheS3Manager;
break;
default:
returnCache = this.levelCacheRef.cacheMemoryManager;
}
this.cacheKeyMap.addToMap(keyArg, returnCache);
return returnCache;
}
/**
* gets the relevant cache to perform a retrieval action on
*/
async getCacheForRetrieveAction() {}
async getCacheForRetrieveAction(keyArg: string): Promise<AbstractCache> {
const done = plugins.smartpromise.defer<AbstractCache>();
const returnCache = this.cacheKeyMap.getByKey(keyArg);
if (!returnCache && this.levelCacheRef.options.persistentCache) {
const checkCache = (cacheArg: AbstractCache) => {
const resultPromise = cacheArg.checkKeyPresence(keyArg);
resultPromise.then(hasKeyArg => {
if (hasKeyArg) {
done.resolve(cacheArg);
}
});
return resultPromise;
};
Promise.all([
checkCache(this.levelCacheRef.cacheMemoryManager),
checkCache(this.levelCacheRef.cacheDiskManager),
checkCache(this.levelCacheRef.cacheMemoryManager)
]).then(() => {
done.resolve(returnCache);
});
} else {
done.resolve(returnCache);
}
return done.promise;
}
}

View File

@ -4,57 +4,99 @@ import { CacheMemoryManager } from './levelcache.classes.cache.memorymanager';
import { CacheS3Manager } from './levelcache.classes.cache.s3manager';
import { CacheEntry } from './levelcache.classes.cacheentry';
import { CacheRouter } from './levelcache.classes.cacherouter';
import { AbstractCache } from './levelcache.abstract.classes.cache';
export interface ILevelCacheConstructorOptions {
maxMemoryStorageInMB: number;
maxDiskStorageInMB: number;
/**
* a unique id when having more than one cache
*/
cacheId: string;
maxMemoryStorageInMB?: number;
maxDiskStorageInMB?: number;
maxS3StorageInMB?: number;
smartbucketConfig?: plugins.smartbucket.ISmartBucketConfig;
diskStoragePath?: string;
s3Config?: plugins.smartbucket.ISmartBucketConfig;
s3BucketName?: string;
forceLevel?: 'memory' | 'disk' | 's3';
expirationInMs?: number;
immutableCache?: boolean;
persistentCache?: boolean;
}
/**
* a leveled cache for storing things for a short time
*/
export class LevelCache {
export class LevelCache extends AbstractCache {
private readyDeferred = plugins.smartpromise.defer<void>();
public ready = this.readyDeferred.promise;
public status: 'active' = 'active'; // artifact of AbstractCache
public cacheRouter = new CacheRouter(this);
public cacheDiskManager = new CacheDiskManager();
public cacheMemoryManager = new CacheMemoryManager();
public cacheS3Manager = new CacheS3Manager();
public cacheDiskManager: CacheDiskManager;
public cacheMemoryManager: CacheMemoryManager;
public cacheS3Manager: CacheS3Manager;
public options: ILevelCacheConstructorOptions;
constructor(optionsArg: ILevelCacheConstructorOptions) {
super();
this.options = optionsArg;
this.init();
}
private processKey(keyArg: string) {
if (!keyArg) {
return plugins.smartunique.shortId();
}
public async init() {
this.cacheDiskManager = new CacheDiskManager(this);
this.cacheMemoryManager = new CacheMemoryManager(this);
this.cacheS3Manager = new CacheS3Manager(this);
await Promise.all([
this.cacheDiskManager.ready,
this.cacheMemoryManager.ready,
this.cacheS3Manager.ready,
]);
this.readyDeferred.resolve();
}
// Blobs
/**
* store a Blob
*/
public async storeBlobByKey(keyArg: string, blob: Buffer) {
keyArg = this.processKey(keyArg);
return keyArg;
// store things
public async storeCacheEntryByKey(keyArg: string, cacheEntryArg: CacheEntry): Promise<void> {
cacheEntryArg.key = keyArg;
const targetCache = await this.cacheRouter.getCacheForStoreAction(keyArg, cacheEntryArg);
await targetCache.storeCacheEntryByKey(keyArg, cacheEntryArg);
}
/**
* retrieve a blob
*/
public async retrieveBlob(keyArg: string): CacheEntry {}
// Cache Entries
// retrieve things
/**
* retrieve cache entry
*/
public async retrieveCacheEntryByKey(): CacheEntry {}
public async retrieveCacheEntryByKey(keyArg: string): Promise<CacheEntry> {
const targetCache = await this.cacheRouter.getCacheForRetrieveAction(keyArg);
if (targetCache) {
const cacheEntry = await targetCache.retrieveCacheEntryByKey(keyArg);
return cacheEntry;
} else {
return null;
}
}
public async checkKeyPresence(keyArg: string): Promise<boolean> {
return plugins.smartpromise.getFirstTrueOrFalse([
this.cacheMemoryManager.checkKeyPresence(keyArg),
this.cacheDiskManager.checkKeyPresence(keyArg),
this.cacheS3Manager.checkKeyPresence(keyArg)
]);
}
// cache maintenance
/**
* cleans the cache
*/
public clean() {}
public async clean(): Promise<void> {
await Promise.all([
this.cacheDiskManager.clean(),
this.cacheDiskManager.clean(),
this.cacheS3Manager.clean(),
]);
}
}

View File

@ -0,0 +1,4 @@
import * as plugins from './levelcache.plugins';
export const packageDir = plugins.path.join(__dirname, '../');
export const nogitDir = plugins.path.join(packageDir, '.nogit/');

View File

@ -8,7 +8,20 @@ import * as lik from '@pushrocks/lik';
import * as smartbucket from '@pushrocks/smartbucket';
import * as smartcache from '@pushrocks/smartcache';
import * as smartfile from '@pushrocks/smartfile';
import * as smartjson from '@pushrocks/smartjson';
import * as smartpromise from '@pushrocks/smartpromise';
import * as smartstring from '@pushrocks/smartstring';
import * as smartunique from '@pushrocks/smartunique';
import * as taskbuffer from '@pushrocks/taskbuffer';
export { lik, smartbucket, smartcache, smartfile, smartstring, smartunique };
export {
lik,
smartbucket,
smartcache,
smartfile,
smartjson,
smartpromise,
smartstring,
smartunique,
taskbuffer,
};

7
tsconfig.json Normal file
View File

@ -0,0 +1,7 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"target": "es2017",
"module": "commonjs"
}
}