feat(core): Enhanced caching solution with optional configurations and improved documentation.

This commit is contained in:
Philipp Kunz 2024-11-24 20:16:25 +01:00
parent e45c662658
commit 75140e9d70
8 changed files with 6659 additions and 3856 deletions

69
changelog.md Normal file
View File

@ -0,0 +1,69 @@
# Changelog
## 2024-11-24 - 3.1.0 - feat(core)
Enhanced caching solution with optional configurations and improved documentation.
- Improved package description and keywords for better discoverability.
- Updated dependencies to latest versions.
- Added strict type checking in TypeScript configurations.
- Enhanced README with detailed usage instructions and examples.
## 2024-05-29 to 2024-02-14 - 3.0.8
Minor configuration updates and documentation changes.
- Updated project description.
- Modified tsconfig setup.
- Adjustments to npmextra.json regarding githost.
## 2024-02-14 - 3.0.7
Core enhancements and bug fixes.
- Fixed updates in the core module to improve stability.
## 2024-02-14 - 3.0.6
Core maintenance and updates.
- Implemented minor fixes in the core module.
## 2023-07-21 - 3.0.5
Addressed core module adjustments.
- Patched core module to rectify issues.
## 2023-07-20 - 3.0.4
Further improvements to core functionality.
- Additional fixes applied to the core component.
## 2023-07-11 to 2023-07-10 - 3.0.3
Organizational and structural changes.
- Transitioned to a new organizational scheme.
## 2023-01-09 - 3.0.2
Core module corrections.
- Resolved various issues within the core module.
## 2022-06-09 - 3.0.1
Continuous enhancements in core functionality.
- Continued bug fixes for core module efficiency.
## 2022-04-04 to 2022-04-02 - 3.0.0
Major release with significant updates.
## 2022-03-22 - 2.0.0
Significant breaking changes in core tech stack.
- BREAKING CHANGE: Transitioned core module to ECMAScript Modules (ESM).
## 2021-05-10 - 1.0.9
Caching improvements and optimization.
- Enhanced caching by properly respecting TTL across all cache levels.
## 2020-02-15 to 2020-02-05 - 1.0.6 to 1.0.1
Initial series of core module fixes and updates.
- Persistent efforts to stabilize and improve core functionalities.

View File

@ -5,22 +5,23 @@
"githost": "code.foss.global", "githost": "code.foss.global",
"gitscope": "push.rocks", "gitscope": "push.rocks",
"gitrepo": "levelcache", "gitrepo": "levelcache",
"description": "A cache that utilizes memory, disk, and S3 for data storage and backup.", "description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
"npmPackagename": "@push.rocks/levelcache", "npmPackagename": "@push.rocks/levelcache",
"license": "MIT", "license": "MIT",
"projectDomain": "push.rocks", "projectDomain": "push.rocks",
"keywords": [ "keywords": [
"caching", "caching",
"data storage",
"memory cache", "memory cache",
"disk cache", "disk cache",
"S3 cache", "S3 storage",
"data backup", "data backup",
"multilevel caching", "multi-level caching",
"performance optimization", "performance",
"persistent storage", "persistent storage",
"data retrieval", "data retrieval",
"node.js", "Node.js",
"typescript" "TypeScript"
] ]
} }
}, },

View File

@ -2,7 +2,7 @@
"name": "@push.rocks/levelcache", "name": "@push.rocks/levelcache",
"version": "3.0.8", "version": "3.0.8",
"private": false, "private": false,
"description": "A cache that utilizes memory, disk, and S3 for data storage and backup.", "description": "A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts", "typings": "dist_ts/index.d.ts",
"type": "module", "type": "module",
@ -18,23 +18,23 @@
"@gitzone/tsbuild": "^2.1.66", "@gitzone/tsbuild": "^2.1.66",
"@gitzone/tsrun": "^1.2.44", "@gitzone/tsrun": "^1.2.44",
"@gitzone/tstest": "^1.0.77", "@gitzone/tstest": "^1.0.77",
"@push.rocks/tapbundle": "^5.0.15", "@push.rocks/tapbundle": "^5.5.3",
"@types/node": "^20.11.17" "@types/node": "^22.9.3"
}, },
"dependencies": { "dependencies": {
"@push.rocks/lik": "^6.0.12", "@push.rocks/lik": "^6.1.0",
"@push.rocks/smartbucket": "^2.0.4", "@push.rocks/smartbucket": "^3.3.3",
"@push.rocks/smartcache": "^1.0.13", "@push.rocks/smartcache": "^1.0.13",
"@push.rocks/smartenv": "^5.0.12", "@push.rocks/smartenv": "^5.0.12",
"@push.rocks/smartexit": "^1.0.21", "@push.rocks/smartexit": "^1.0.23",
"@push.rocks/smartfile": "^11.0.4", "@push.rocks/smartfile": "^11.0.21",
"@push.rocks/smartjson": "^5.0.10", "@push.rocks/smartjson": "^5.0.20",
"@push.rocks/smartpath": "^5.0.11", "@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartpromise": "^4.0.4",
"@push.rocks/smartstring": "^4.0.13", "@push.rocks/smartstring": "^4.0.15",
"@push.rocks/smartunique": "^3.0.6", "@push.rocks/smartunique": "^3.0.9",
"@push.rocks/taskbuffer": "^3.1.7", "@push.rocks/taskbuffer": "^3.1.7",
"@tsclass/tsclass": "^4.0.51" "@tsclass/tsclass": "^4.1.2"
}, },
"files": [ "files": [
"ts/**/*", "ts/**/*",
@ -53,16 +53,17 @@
], ],
"keywords": [ "keywords": [
"caching", "caching",
"data storage",
"memory cache", "memory cache",
"disk cache", "disk cache",
"S3 cache", "S3 storage",
"data backup", "data backup",
"multilevel caching", "multi-level caching",
"performance optimization", "performance",
"persistent storage", "persistent storage",
"data retrieval", "data retrieval",
"node.js", "Node.js",
"typescript" "TypeScript"
], ],
"homepage": "https://code.foss.global/push.rocks/levelcache", "homepage": "https://code.foss.global/push.rocks/levelcache",
"repository": { "repository": {

10236
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

149
readme.md
View File

@ -1,8 +1,9 @@
```markdown
# @push.rocks/levelcache # @push.rocks/levelcache
a cache that uses memory/disk/s3 as backup A cache that utilizes memory, disk, and S3 for data storage and backup.
## Install ## Install
To install `@push.rocks/levelcache`, you can use npm or yarn as follows: To install `@push.rocks/levelcache`, you can use npm or yarn:
```bash ```bash
npm install @push.rocks/levelcache --save npm install @push.rocks/levelcache --save
@ -12,99 +13,155 @@ or
yarn add @push.rocks/levelcache yarn add @push.rocks/levelcache
``` ```
This will add `@push.rocks/levelcache` to your project's dependencies. This installs `@push.rocks/levelcache` and adds it to your project's dependencies.
## Usage ## Usage
`@push.rocks/levelcache` provides a versatile caching solution that leverages memory, disk, and Amazon S3 for data storage, ensuring both speed and durability. By prioritizing data storage based on the size and storage limits you define, it efficiently manages where to store your cache entries. `@push.rocks/levelcache` provides a comprehensive solution for multi-level caching that takes advantage of memory, disk, and Amazon S3 storage, making it a versatile tool for data caching and backup. The package is built with TypeScript, enabling strict type checks and better development experience. Below, we'll explore how to effectively employ `@push.rocks/levelcache` in your projects, discussing its features and demonstrating its usage with code examples.
### Getting Started ### 1. Overview
First, you need to import `LevelCache` and `CacheEntry` from the package: The `LevelCache` class handles all cache operations. It decides where to store data based on pre-configured thresholds corresponding to the data size and the total storage capacity allocated for each storage type (memory/disk/S3). This mechanism optimizes both speed and persistence, allowing for efficient data storage and retrieval.
### 2. Getting Started: Initialization
To use `@push.rocks/levelcache`, you'll need to import the main classes: `LevelCache` and `CacheEntry`. `LevelCache` is the primary class, while `CacheEntry` represents individual pieces of cached data.
```typescript ```typescript
import { LevelCache, CacheEntry } from '@push.rocks/levelcache'; import { LevelCache, CacheEntry } from '@push.rocks/levelcache';
``` ```
### Initializing a Cache #### Initialization with Optional Configurations
You initialize a new cache by creating an instance of `LevelCache`. You can specify options such as memory and disk size limits, S3 configurations, and more. To create a cache, instantiate the `LevelCache` class with desired configurations. You can specify the limits for memory and disk storage, setup S3 configurations if needed, and more.
```typescript ```typescript
const myCache = new LevelCache({ const myCache = new LevelCache({
cacheId: 'myUniqueCacheId', // A unique ID for your cache cacheId: 'myUniqueCacheId', // Unique ID for cache delineation
maxMemoryStorageInMB: 10, // Optional - default is 0.5 MB maxMemoryStorageInMB: 10, // Maximum memory use in MB (default 0.5 MB)
maxDiskStorageInMB: 100, // Optional - default is 10 MB maxDiskStorageInMB: 100, // Maximum disk space in MB (default 10 MB)
diskStoragePath: './myCache', // Optional - default uses a '.nogit' directory diskStoragePath: './myCache', // Path for storing disk cache; default is '.nogit'
// Specify S3 configurations if you want to use S3 as a storage level
s3Config: { s3Config: {
accessKeyId: 'yourAccessKeyId', // Your S3 access key accessKeyId: 'yourAccessKeyId', // AWS S3 access key
secretAccessKey: 'yourSecretAccessKey', // Your S3 secret key secretAccessKey: 'yourSecretAccessKey', // Corresponding secret key
region: 'us-west-2' // The region your S3 bucket is in region: 'us-west-2' // AWS region, e.g., 'us-west-2'
}, },
s3BucketName: 'myBucketName', // The name of your S3 bucket s3BucketName: 'myBucketName', // Designated name for S3 bucket
immutableCache: true, // Optional - if cache entries should be immutable immutableCache: false, // Whether stored cache entries should remain unaltered
persistentCache: true, // Optional - if cache should persist between restarts persistentCache: true, // Should the cache persist upon restarts
}); });
``` ```
### Storing and Retrieving Cached Data ### 3. Storing and Retrieving Data
To store data, you create a `CacheEntry` and use `storeCacheEntryByKey` method. Retrieving data is done using the `retrieveCacheEntryByKey` method. `LevelCache` methods enable seamless data storage and retrieval, handling complexity under the hood.
#### Storing Data
Create a `CacheEntry` specifying the data content and time-to-live (`ttl`). Use `storeCacheEntryByKey` to add this entry to the cache.
```typescript ```typescript
async function cacheOperations() { async function storeData() {
// Ensure the cache is ready // Wait for cache to be ready before operations
await myCache.ready; await myCache.ready;
// Create a cache entry const entryContents = Buffer.from('Caching this data');
const myCacheEntry = new CacheEntry({ const myCacheEntry = new CacheEntry({
ttl: 3600000, // Time-to-live in milliseconds ttl: 7200000, // Time-to-live in milliseconds (2 hours)
contents: Buffer.from('This is some data to cache'), // The data to cache contents: entryContents,
}); });
// Store the cache entry // Storing the cache entry associated with a specific key
await myCache.storeCacheEntryByKey('myDataKey', myCacheEntry); await myCache.storeCacheEntryByKey('someDataKey', myCacheEntry);
}
```
// Retrieve the cache entry #### Retrieving Data
const retrievedEntry = await myCache.retrieveCacheEntryByKey('myDataKey');
if(retrievedEntry) { Retrieve stored data using `retrieveCacheEntryByKey`. The retrieved `CacheEntry` will give access to the original data.
console.log(retrievedEntry.contents.toString()); // Logs: This is some data to cache
```typescript
async function retrieveData() {
const retrievedEntry = await myCache.retrieveCacheEntryByKey('someDataKey');
if (retrievedEntry) {
const data = retrievedEntry.contents.toString();
console.log(data); // Expected output: Caching this data
} else {
console.log('Data not found or expired.');
} }
} }
cacheOperations();
``` ```
### Deleting Cache Entries ### 4. Key Management: Updating and Deleting
You can delete entries from the cache using `deleteCacheEntryByKey` method: #### Deleting Cache Entries
Remove entries with `deleteCacheEntryByKey`, enabling clean cache management.
```typescript ```typescript
await myCache.deleteCacheEntryByKey('myDataKey'); async function deleteData() {
// Removes an entry using its unique key identifier
await myCache.deleteCacheEntryByKey('someDataKey');
}
``` ```
### Cache Cleanup ### 5. Cache Cleaning
The library provides mechanisms to clean outdated entries or flush the entire cache. Often, managing storage limits or removing outdated data becomes essential. The library supports these scenarios.
#### Automated Cleaning
While cache entries will naturally expire with `ttl` values, you can force-remove outdated entries.
```typescript ```typescript
// Clean outdated cache entries // Clean outdated or expired entries
await myCache.cleanOutdated(); await myCache.cleanOutdated();
```
// Clean all cache entries #### Full Cache Reset
Clear all entries, efficiently resetting your cache storage.
```typescript
// Flush entire cache content
await myCache.cleanAll(); await myCache.cleanAll();
``` ```
### Advanced Use Cases ### 6. Configuring and Managing Advanced Use Cases
For more complex caching requirements, such as custom cache routing or handling larger datasets with varying priorities, you can dive deeper into the library's API to customize your caching strategy. This includes configuring thresholds for when to use each storage level and managing cache immutability and persistence. The flexible nature of `@push.rocks/levelcache` grants additional customization suited for more advanced requirements.
For further information and more in-depth examples, you're encouraged to explore the source code and experiment with configuring `LevelCache` to best suit your application's needs. #### Custom Route Management
`@push.rocks/levelcache` blends the speed of in-memory caching with the durability and scalability of disk and S3-based caching, making it a comprehensive solution for your caching needs. For certain demands, you might want to specify distinct data handling policies or routing logic.
By carefully considering your application's data access patterns, size constraints, and persistence requirements, you can leverage `@push.rocks/levelcache` to significantly improve performance and efficiency. - Adjust S3 handling, size thresholds, or immutability options dynamically.
- Utilize internal API expansions defined within the library for fine-grained operations.
#### Handling Large Datasets
Tailor the cache levels (memory, disk, S3) to accommodate higher loads:
```typescript
const largeDatasetCache = new LevelCache({
cacheId: 'largeDatasetCache',
// Customize limits and behavior for particular patterns
maxMemoryStorageInMB: 1024, // 1 GB memory allocation
maxDiskStorageInMB: 2048, // 2 GB disk space allowance
maxS3StorageInMB: 10240, // 10 GB S3 backup buffering
});
```
With intelligent routing and management embedded, `LevelCache` ensures optimal trade-offs between speed and stability.
### Conclusion
By adapting to bespoke caching styles and leveraging extensive storage structures (in-memory, on-disk, and cloud-based), `@push.rocks/levelcache` can handle varied data caching use-cases with ease. Whether you're aiming for top-tier speed for volatile data or need extended persistence for critical datasets, configure `LevelCache` to excellently complement your operational context.
Explore the package further through testing and customization, ensuring you're getting the most benefit from integrated features and storage mechanisms. The robustness of `@push.rocks/levelcache` consistently optimizes the caching and retrieval process across different runtime environments.
We recommend examining your own application's storage behavior taxonomy; this helps frame caching strategies that consider both speed and durability requirements. Integrate `@push.rocks/levelcache` as a billing cornerstone of your systems architecture built on TypeScript and Node.js, embracing luxury control over resource use and performance elevation.
```
## License and Legal Information ## License and Legal Information

View File

@ -35,7 +35,7 @@ tap.test('should respect ttl', async (tools) => {
); );
const result = await testLevelCache.retrieveCacheEntryByKey('mykey'); const result = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result.contents.toString()).toEqual('heythere'); expect(result.contents.toString()).toEqual('heythere');
await tools.delayFor(1100); await tools!.delayFor(1100);
const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey'); const result2 = await testLevelCache.retrieveCacheEntryByKey('mykey');
expect(result2).toBeNull(); expect(result2).toBeNull();
}); });

View File

@ -1,8 +1,8 @@
/** /**
* autocreated commitinfo by @pushrocks/commitinfo * autocreated commitinfo by @push.rocks/commitinfo
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/levelcache', name: '@push.rocks/levelcache',
version: '3.0.8', version: '3.1.0',
description: 'a cache that uses memory/disk/s3 as backup' description: 'A versatile caching solution offering multi-level storage utilizing memory, disk, and Amazon S3 for efficient data management and backup.'
} }

View File

@ -6,7 +6,8 @@
"module": "NodeNext", "module": "NodeNext",
"moduleResolution": "NodeNext", "moduleResolution": "NodeNext",
"esModuleInterop": true, "esModuleInterop": true,
"verbatimModuleSyntax": true "verbatimModuleSyntax": true,
"strict": true
}, },
"exclude": [ "exclude": [
"dist_*/**/*.d.ts" "dist_*/**/*.d.ts"