Juergen Kunz
820f84ee61
- Fix ES client v8+ API: use document/doc instead of body for index/update operations - Add type assertions (as any) for ES client ILM, template, and search APIs - Fix strict null checks with proper undefined handling (nullish coalescing) - Fix MetricsCollector interface to match required method signatures - Fix Logger.error signature compatibility in plugins - Resolve TermsQuery type index signature conflict - Remove sourceMap from tsconfig (handled by tsbuild with inlineSourceMap)
472 lines
15 KiB
TypeScript
472 lines
15 KiB
TypeScript
/**
|
|
* Comprehensive KV Store Example
|
|
*
|
|
* Demonstrates distributed key-value storage with TTL and caching
|
|
*/
|
|
|
|
import {
|
|
createConfig,
|
|
ElasticsearchConnectionManager,
|
|
LogLevel,
|
|
KVStore,
|
|
} from '../../index.js';
|
|
|
|
interface UserSession {
|
|
userId: string;
|
|
username: string;
|
|
email: string;
|
|
roles: string[];
|
|
loginAt: Date;
|
|
lastActivityAt: Date;
|
|
metadata: {
|
|
ip: string;
|
|
userAgent: string;
|
|
};
|
|
}
|
|
|
|
interface CacheData {
|
|
query: string;
|
|
results: unknown[];
|
|
computedAt: Date;
|
|
ttl: number;
|
|
}
|
|
|
|
async function main() {
|
|
console.log('=== KV Store Example ===\n');
|
|
|
|
// ============================================================================
|
|
// Step 1: Configuration
|
|
// ============================================================================
|
|
|
|
console.log('Step 1: Configuring Elasticsearch connection...');
|
|
const config = createConfig()
|
|
.fromEnv()
|
|
.nodes(process.env.ELASTICSEARCH_URL || 'http://localhost:9200')
|
|
.basicAuth(
|
|
process.env.ELASTICSEARCH_USERNAME || 'elastic',
|
|
process.env.ELASTICSEARCH_PASSWORD || 'changeme'
|
|
)
|
|
.timeout(30000)
|
|
.retries(3)
|
|
.logLevel(LogLevel.INFO)
|
|
.enableMetrics(true)
|
|
.build();
|
|
|
|
// ============================================================================
|
|
// Step 2: Initialize Connection
|
|
// ============================================================================
|
|
|
|
console.log('Step 2: Initializing connection manager...');
|
|
const connectionManager = ElasticsearchConnectionManager.getInstance(config);
|
|
await connectionManager.initialize();
|
|
console.log('✓ Connection manager initialized\n');
|
|
|
|
// ============================================================================
|
|
// Step 3: Basic KV Operations
|
|
// ============================================================================
|
|
|
|
console.log('Step 3: Basic key-value operations...');
|
|
const basicKV = new KVStore<string>({
|
|
index: 'kv-basic',
|
|
enableCache: true,
|
|
cacheMaxSize: 1000,
|
|
});
|
|
|
|
await basicKV.initialize();
|
|
|
|
// Set a value
|
|
await basicKV.set('user:1:name', 'Alice Johnson');
|
|
await basicKV.set('user:2:name', 'Bob Smith');
|
|
await basicKV.set('user:3:name', 'Charlie Brown');
|
|
|
|
// Get a value
|
|
const result = await basicKV.get('user:1:name');
|
|
console.log(` Retrieved: ${result.value} (cache hit: ${result.cacheHit})`);
|
|
|
|
// Get again (should hit cache)
|
|
const cachedResult = await basicKV.get('user:1:name');
|
|
console.log(` Retrieved: ${cachedResult.value} (cache hit: ${cachedResult.cacheHit})`);
|
|
|
|
// Check existence
|
|
const exists = await basicKV.exists('user:1:name');
|
|
console.log(` Key exists: ${exists}`);
|
|
|
|
// Delete a key
|
|
await basicKV.delete('user:3:name');
|
|
const deletedExists = await basicKV.exists('user:3:name');
|
|
console.log(` Deleted key exists: ${deletedExists}`);
|
|
|
|
console.log('✓ Basic operations complete\n');
|
|
|
|
// ============================================================================
|
|
// Step 4: TTL Support
|
|
// ============================================================================
|
|
|
|
console.log('Step 4: TTL (Time-To-Live) support...');
|
|
const ttlKV = new KVStore<UserSession>({
|
|
index: 'kv-sessions',
|
|
defaultTTL: 3600, // 1 hour default
|
|
enableCache: true,
|
|
enableExpirationCleanup: true,
|
|
cleanupIntervalSeconds: 60,
|
|
});
|
|
|
|
await ttlKV.initialize();
|
|
|
|
// Set session with 5-second TTL
|
|
const session: UserSession = {
|
|
userId: 'user-123',
|
|
username: 'alice',
|
|
email: 'alice@example.com',
|
|
roles: ['user', 'admin'],
|
|
loginAt: new Date(),
|
|
lastActivityAt: new Date(),
|
|
metadata: {
|
|
ip: '192.168.1.100',
|
|
userAgent: 'Mozilla/5.0',
|
|
},
|
|
};
|
|
|
|
await ttlKV.set('session:alice-token-xyz', session, { ttl: 5 });
|
|
console.log(' Session stored with 5-second TTL');
|
|
|
|
// Get immediately
|
|
const sessionResult = await ttlKV.get('session:alice-token-xyz');
|
|
console.log(` Session retrieved: ${sessionResult.value?.username}`);
|
|
console.log(` Expires at: ${sessionResult.expiresAt?.toISOString()}`);
|
|
|
|
// Wait 6 seconds and try again
|
|
console.log(' Waiting 6 seconds for expiration...');
|
|
await new Promise((resolve) => setTimeout(resolve, 6000));
|
|
|
|
const expiredResult = await ttlKV.get('session:alice-token-xyz');
|
|
console.log(` After expiration - exists: ${expiredResult.exists}`);
|
|
|
|
console.log('✓ TTL support demonstrated\n');
|
|
|
|
// ============================================================================
|
|
// Step 5: Batch Operations
|
|
// ============================================================================
|
|
|
|
console.log('Step 5: Batch operations...');
|
|
const batchKV = new KVStore<CacheData>({
|
|
index: 'kv-cache',
|
|
enableCache: true,
|
|
cacheMaxSize: 5000,
|
|
});
|
|
|
|
await batchKV.initialize();
|
|
|
|
// Batch set
|
|
const cacheEntries = [
|
|
{
|
|
key: 'cache:query:1',
|
|
value: {
|
|
query: 'SELECT * FROM users',
|
|
results: [{ id: 1, name: 'Alice' }],
|
|
computedAt: new Date(),
|
|
ttl: 300,
|
|
},
|
|
options: { ttl: 300 },
|
|
},
|
|
{
|
|
key: 'cache:query:2',
|
|
value: {
|
|
query: 'SELECT * FROM products',
|
|
results: [{ id: 1, name: 'Product A' }],
|
|
computedAt: new Date(),
|
|
ttl: 300,
|
|
},
|
|
options: { ttl: 300 },
|
|
},
|
|
{
|
|
key: 'cache:query:3',
|
|
value: {
|
|
query: 'SELECT * FROM orders',
|
|
results: [{ id: 1, total: 100 }],
|
|
computedAt: new Date(),
|
|
ttl: 300,
|
|
},
|
|
options: { ttl: 300 },
|
|
},
|
|
];
|
|
|
|
const msetResult = await batchKV.mset(cacheEntries);
|
|
console.log(` Batch set: ${msetResult.successful} successful, ${msetResult.failed} failed`);
|
|
|
|
// Batch get
|
|
const mgetResult = await batchKV.mget([
|
|
'cache:query:1',
|
|
'cache:query:2',
|
|
'cache:query:3',
|
|
'cache:query:999', // Doesn't exist
|
|
]);
|
|
|
|
console.log(` Batch get: ${mgetResult.found} found, ${mgetResult.notFound} not found`);
|
|
console.log(` Cache hits: ${mgetResult.cacheHits}`);
|
|
|
|
// Batch delete
|
|
const mdeleteResult = await batchKV.mdelete(['cache:query:1', 'cache:query:2']);
|
|
console.log(
|
|
` Batch delete: ${mdeleteResult.successful} successful, ${mdeleteResult.failed} failed`
|
|
);
|
|
|
|
console.log('✓ Batch operations complete\n');
|
|
|
|
// ============================================================================
|
|
// Step 6: Key Scanning
|
|
// ============================================================================
|
|
|
|
console.log('Step 6: Key scanning with patterns...');
|
|
const scanKV = new KVStore<string>({
|
|
index: 'kv-scan',
|
|
enableCache: false,
|
|
});
|
|
|
|
await scanKV.initialize();
|
|
|
|
// Create test data
|
|
await scanKV.set('user:1:profile', 'Profile 1');
|
|
await scanKV.set('user:2:profile', 'Profile 2');
|
|
await scanKV.set('user:3:profile', 'Profile 3');
|
|
await scanKV.set('product:1:info', 'Product Info 1');
|
|
await scanKV.set('product:2:info', 'Product Info 2');
|
|
|
|
// Scan all user profiles
|
|
const userScan = await scanKV.scan({
|
|
pattern: 'user:*:profile',
|
|
limit: 10,
|
|
includeValues: false,
|
|
});
|
|
|
|
console.log(` User profiles found: ${userScan.keys.length}`);
|
|
console.log(` Keys: ${userScan.keys.join(', ')}`);
|
|
|
|
// Scan all products with values
|
|
const productScan = await scanKV.scan({
|
|
pattern: 'product:*',
|
|
limit: 10,
|
|
includeValues: true,
|
|
});
|
|
|
|
console.log(` Products found: ${productScan.keys.length}`);
|
|
console.log(` First product: ${productScan.values?.[0]}`);
|
|
|
|
// Scan with pagination
|
|
console.log(' Paginated scan:');
|
|
let cursor: string | undefined;
|
|
let page = 1;
|
|
|
|
do {
|
|
const result = await scanKV.scan({
|
|
limit: 2,
|
|
cursor,
|
|
includeValues: false,
|
|
});
|
|
|
|
console.log(` Page ${page}: ${result.keys.length} keys`);
|
|
cursor = result.nextCursor;
|
|
page++;
|
|
|
|
if (!result.hasMore) break;
|
|
} while (cursor && page <= 3);
|
|
|
|
console.log('✓ Key scanning complete\n');
|
|
|
|
// ============================================================================
|
|
// Step 7: Cache Eviction Policies
|
|
// ============================================================================
|
|
|
|
console.log('Step 7: Cache eviction policies...');
|
|
|
|
// LRU (Least Recently Used)
|
|
console.log(' Testing LRU eviction policy...');
|
|
const lruKV = new KVStore<number>({
|
|
index: 'kv-eviction-lru',
|
|
enableCache: true,
|
|
cacheMaxSize: 3,
|
|
cacheEvictionPolicy: 'lru',
|
|
});
|
|
|
|
await lruKV.initialize();
|
|
|
|
await lruKV.set('key1', 1);
|
|
await lruKV.set('key2', 2);
|
|
await lruKV.set('key3', 3);
|
|
|
|
// Access key1 (make it recently used)
|
|
await lruKV.get('key1');
|
|
|
|
// Add key4 (should evict key2, the least recently used)
|
|
await lruKV.set('key4', 4);
|
|
|
|
const stats = lruKV.getStats();
|
|
console.log(` Cache size: ${stats.cacheStats?.size}/${stats.cacheStats?.maxSize}`);
|
|
console.log(` Evictions: ${stats.cacheStats?.evictions}`);
|
|
|
|
// LFU (Least Frequently Used)
|
|
console.log(' Testing LFU eviction policy...');
|
|
const lfuKV = new KVStore<number>({
|
|
index: 'kv-eviction-lfu',
|
|
enableCache: true,
|
|
cacheMaxSize: 3,
|
|
cacheEvictionPolicy: 'lfu',
|
|
});
|
|
|
|
await lfuKV.initialize();
|
|
|
|
await lfuKV.set('key1', 1);
|
|
await lfuKV.set('key2', 2);
|
|
await lfuKV.set('key3', 3);
|
|
|
|
// Access key1 multiple times
|
|
await lfuKV.get('key1');
|
|
await lfuKV.get('key1');
|
|
await lfuKV.get('key1');
|
|
|
|
// Add key4 (should evict key2 or key3, the least frequently used)
|
|
await lfuKV.set('key4', 4);
|
|
|
|
const lfuStats = lfuKV.getStats();
|
|
console.log(` Cache size: ${lfuStats.cacheStats?.size}/${lfuStats.cacheStats?.maxSize}`);
|
|
console.log(` Evictions: ${lfuStats.cacheStats?.evictions}`);
|
|
|
|
console.log('✓ Cache eviction policies demonstrated\n');
|
|
|
|
// ============================================================================
|
|
// Step 8: Optimistic Concurrency
|
|
// ============================================================================
|
|
|
|
console.log('Step 8: Optimistic concurrency control...');
|
|
const concurrencyKV = new KVStore<{ count: number }>({
|
|
index: 'kv-concurrency',
|
|
enableOptimisticConcurrency: true,
|
|
enableCache: false,
|
|
});
|
|
|
|
await concurrencyKV.initialize();
|
|
|
|
// Set initial value
|
|
const initial = await concurrencyKV.set('counter', { count: 0 });
|
|
console.log(` Initial version: seq_no=${initial.version?.seqNo}`);
|
|
|
|
// Update with correct version
|
|
const update1 = await concurrencyKV.set('counter', { count: 1 }, {
|
|
ifSeqNo: initial.version?.seqNo,
|
|
ifPrimaryTerm: initial.version?.primaryTerm,
|
|
});
|
|
|
|
console.log(` Update 1 success: ${update1.success}`);
|
|
|
|
// Try to update with old version (should fail)
|
|
const update2 = await concurrencyKV.set('counter', { count: 999 }, {
|
|
ifSeqNo: initial.version?.seqNo, // Old version
|
|
ifPrimaryTerm: initial.version?.primaryTerm,
|
|
});
|
|
|
|
console.log(` Update 2 with old version success: ${update2.success}`);
|
|
if (!update2.success) {
|
|
console.log(` Error: ${update2.error?.type} - ${update2.error?.reason}`);
|
|
}
|
|
|
|
console.log('✓ Optimistic concurrency demonstrated\n');
|
|
|
|
// ============================================================================
|
|
// Step 9: Compression
|
|
// ============================================================================
|
|
|
|
console.log('Step 9: Automatic compression for large values...');
|
|
const compressionKV = new KVStore<{ data: string }>({
|
|
index: 'kv-compression',
|
|
enableCompression: true,
|
|
compressionThreshold: 100, // 100 bytes
|
|
enableCache: false,
|
|
});
|
|
|
|
await compressionKV.initialize();
|
|
|
|
// Small value (no compression)
|
|
await compressionKV.set('small', { data: 'Hello' });
|
|
|
|
// Large value (will be compressed)
|
|
const largeData = 'x'.repeat(1000);
|
|
await compressionKV.set('large', { data: largeData });
|
|
|
|
// Retrieve both
|
|
const smallResult = await compressionKV.get('small');
|
|
const largeResult = await compressionKV.get('large');
|
|
|
|
console.log(` Small value retrieved: ${smallResult.value?.data.substring(0, 10)}...`);
|
|
console.log(` Large value retrieved: ${largeResult.value?.data.substring(0, 10)}... (length: ${largeResult.value?.data.length})`);
|
|
|
|
console.log('✓ Compression demonstrated\n');
|
|
|
|
// ============================================================================
|
|
// Step 10: Statistics
|
|
// ============================================================================
|
|
|
|
console.log('Step 10: KV Store statistics...\n');
|
|
|
|
const finalStats = basicKV.getStats();
|
|
|
|
console.log('Basic KV Store Statistics:');
|
|
console.log(` Total keys: ${finalStats.totalKeys}`);
|
|
console.log(` Total gets: ${finalStats.totalGets}`);
|
|
console.log(` Total sets: ${finalStats.totalSets}`);
|
|
console.log(` Total deletes: ${finalStats.totalDeletes}`);
|
|
console.log(` Total scans: ${finalStats.totalScans}`);
|
|
console.log(` Total expired: ${finalStats.totalExpired}`);
|
|
console.log(` Avg get duration: ${finalStats.avgGetDurationMs.toFixed(2)}ms`);
|
|
console.log(` Avg set duration: ${finalStats.avgSetDurationMs.toFixed(2)}ms`);
|
|
console.log(` Avg delete duration: ${finalStats.avgDeleteDurationMs.toFixed(2)}ms`);
|
|
|
|
if (finalStats.cacheStats) {
|
|
console.log('\n Cache Statistics:');
|
|
console.log(` Size: ${finalStats.cacheStats.size}/${finalStats.cacheStats.maxSize}`);
|
|
console.log(` Hits: ${finalStats.cacheStats.hits}`);
|
|
console.log(` Misses: ${finalStats.cacheStats.misses}`);
|
|
console.log(` Hit ratio: ${(finalStats.cacheStats.hitRatio * 100).toFixed(2)}%`);
|
|
console.log(` Evictions: ${finalStats.cacheStats.evictions}`);
|
|
console.log(` Memory usage: ${(finalStats.cacheStats.memoryUsage / 1024).toFixed(2)} KB`);
|
|
}
|
|
|
|
console.log();
|
|
|
|
// ============================================================================
|
|
// Step 11: Cleanup
|
|
// ============================================================================
|
|
|
|
console.log('Step 11: Cleanup...');
|
|
|
|
await basicKV.destroy();
|
|
await ttlKV.destroy();
|
|
await batchKV.destroy();
|
|
await scanKV.destroy();
|
|
await lruKV.destroy();
|
|
await lfuKV.destroy();
|
|
await concurrencyKV.destroy();
|
|
await compressionKV.destroy();
|
|
|
|
await connectionManager.destroy();
|
|
console.log('✓ Cleanup complete\n');
|
|
|
|
console.log('=== KV Store Example Complete ===');
|
|
console.log('\nKey Features Demonstrated:');
|
|
console.log(' ✓ Basic get/set/delete operations');
|
|
console.log(' ✓ TTL (Time-To-Live) with automatic expiration');
|
|
console.log(' ✓ In-memory caching with hit/miss tracking');
|
|
console.log(' ✓ Batch operations (mget, mset, mdelete)');
|
|
console.log(' ✓ Key scanning with wildcard patterns');
|
|
console.log(' ✓ Cache eviction policies (LRU, LFU, FIFO, TTL)');
|
|
console.log(' ✓ Optimistic concurrency control');
|
|
console.log(' ✓ Automatic compression for large values');
|
|
console.log(' ✓ Comprehensive statistics');
|
|
console.log(' ✓ Cursor-based pagination');
|
|
}
|
|
|
|
// Run the example
|
|
main().catch((error) => {
|
|
console.error('Example failed:', error);
|
|
process.exit(1);
|
|
});
|