import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as qenv from '@push.rocks/qenv'; import * as smartbucket from '@push.rocks/smartbucket'; import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js'; import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js'; import type { IStorageBackend } from '../ts/core/interfaces.core.js'; import { generateTestRunId } from './helpers/registry.js'; const testQenv = new qenv.Qenv('./', './.nogit'); // ============================================================================ // Test State // ============================================================================ let cache: UpstreamCache; let storageBackend: IStorageBackend; let s3Bucket: smartbucket.Bucket; let smartBucket: smartbucket.SmartBucket; let testRunId: string; let bucketName: string; // ============================================================================ // Helper Functions // ============================================================================ function createFetchContext(overrides?: Partial): IUpstreamFetchContext { // Use resource name as path to ensure unique cache keys const resource = overrides?.resource || 'lodash'; return { protocol: 'npm', resource, resourceType: 'packument', path: `/${resource}`, method: 'GET', headers: {}, query: {}, ...overrides, }; } // ============================================================================ // Setup // ============================================================================ tap.test('setup: should create S3 storage backend', async () => { testRunId = generateTestRunId(); bucketName = `test-ucache-${testRunId.substring(0, 8)}`; const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'); const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY'); const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT'); const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT'); smartBucket = new smartbucket.SmartBucket({ accessKey: s3AccessKey || 'minioadmin', accessSecret: s3SecretKey || 'minioadmin', endpoint: s3Endpoint || 'localhost', port: parseInt(s3Port || '9000', 10), useSsl: false, }); s3Bucket = await smartBucket.createBucket(bucketName); // Create storage backend adapter storageBackend = { getObject: async (key: string): Promise => { try { // fastGet returns Buffer directly (or undefined if not found) const data = await s3Bucket.fastGet({ path: key }); if (!data) { return null; } return data; } catch (error) { // fastGet throws if object doesn't exist return null; } }, putObject: async (key: string, data: Buffer): Promise => { await s3Bucket.fastPut({ path: key, contents: data, overwrite: true }); }, deleteObject: async (key: string): Promise => { await s3Bucket.fastRemove({ path: key }); }, listObjects: async (prefix: string): Promise => { const paths: string[] = []; for await (const path of s3Bucket.listAllObjects(prefix)) { paths.push(path); } return paths; }, }; expect(storageBackend).toBeTruthy(); }); tap.test('setup: should create UpstreamCache with S3 storage', async () => { cache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 300 }, 10000, storageBackend ); expect(cache.isEnabled()).toBeTrue(); expect(cache.hasStorage()).toBeTrue(); }); // ============================================================================ // Basic Cache Operations // ============================================================================ tap.test('cache: should store cache entry in S3', async () => { const context = createFetchContext({ resource: 'store-test' }); const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' })); const upstreamUrl = 'https://registry.npmjs.org'; await cache.set( context, testData, 'application/json', { 'etag': '"abc123"' }, 'npmjs', upstreamUrl ); // Verify in S3 const stats = cache.getStats(); expect(stats.totalEntries).toBeGreaterThanOrEqual(1); }); tap.test('cache: should retrieve cache entry from S3', async () => { const context = createFetchContext({ resource: 'retrieve-test' }); const testData = Buffer.from('retrieve test data'); const upstreamUrl = 'https://registry.npmjs.org'; await cache.set( context, testData, 'application/octet-stream', {}, 'npmjs', upstreamUrl ); const entry = await cache.get(context, upstreamUrl); expect(entry).toBeTruthy(); expect(entry!.data.toString()).toEqual('retrieve test data'); expect(entry!.contentType).toEqual('application/octet-stream'); expect(entry!.upstreamId).toEqual('npmjs'); }); // ============================================================================ // Multi-Upstream URL Tests // ============================================================================ tap.test('cache: should include upstream URL in cache path', async () => { const context = createFetchContext({ resource: 'url-path-test' }); const testData = Buffer.from('url path test'); await cache.set( context, testData, 'text/plain', {}, 'npmjs', 'https://registry.npmjs.org' ); // The cache key should include the escaped URL const entry = await cache.get(context, 'https://registry.npmjs.org'); expect(entry).toBeTruthy(); expect(entry!.data.toString()).toEqual('url path test'); }); tap.test('cache: should handle multiple upstreams with different URLs', async () => { const context = createFetchContext({ resource: '@company/private-pkg' }); // Store from private upstream const privateData = Buffer.from('private package data'); await cache.set( context, privateData, 'application/json', {}, 'private-npm', 'https://npm.company.com' ); // Store from public upstream (same resource name, different upstream) const publicData = Buffer.from('public package data'); await cache.set( context, publicData, 'application/json', {}, 'public-npm', 'https://registry.npmjs.org' ); // Retrieve from private - should get private data const privateEntry = await cache.get(context, 'https://npm.company.com'); expect(privateEntry).toBeTruthy(); expect(privateEntry!.data.toString()).toEqual('private package data'); expect(privateEntry!.upstreamId).toEqual('private-npm'); // Retrieve from public - should get public data const publicEntry = await cache.get(context, 'https://registry.npmjs.org'); expect(publicEntry).toBeTruthy(); expect(publicEntry!.data.toString()).toEqual('public package data'); expect(publicEntry!.upstreamId).toEqual('public-npm'); }); // ============================================================================ // TTL and Expiration Tests // ============================================================================ tap.test('cache: should respect TTL expiration', async () => { // Create cache with very short TTL const shortTtlCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 1, // 1 second TTL staleWhileRevalidate: false, staleMaxAgeSeconds: 0, immutableTtlSeconds: 1, negativeCacheTtlSeconds: 1, }, 1000, storageBackend ); const context = createFetchContext({ resource: 'ttl-test' }); const testData = Buffer.from('expires soon'); await shortTtlCache.set( context, testData, 'text/plain', {}, 'test-upstream', 'https://test.example.com' ); // Should exist immediately let entry = await shortTtlCache.get(context, 'https://test.example.com'); expect(entry).toBeTruthy(); // Wait for expiration await new Promise(resolve => setTimeout(resolve, 1500)); // Should be expired now entry = await shortTtlCache.get(context, 'https://test.example.com'); expect(entry).toBeNull(); shortTtlCache.stop(); }); tap.test('cache: should serve stale content during stale-while-revalidate window', async () => { const staleCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 1, // 1 second fresh staleWhileRevalidate: true, staleMaxAgeSeconds: 60, // 60 seconds stale window immutableTtlSeconds: 1, negativeCacheTtlSeconds: 1, }, 1000, storageBackend ); const context = createFetchContext({ resource: 'stale-test' }); const testData = Buffer.from('stale but usable'); await staleCache.set( context, testData, 'text/plain', {}, 'stale-upstream', 'https://stale.example.com' ); // Wait for fresh period to expire await new Promise(resolve => setTimeout(resolve, 1500)); // Should still be available but marked as stale const entry = await staleCache.get(context, 'https://stale.example.com'); expect(entry).toBeTruthy(); expect(entry!.stale).toBeTrue(); expect(entry!.data.toString()).toEqual('stale but usable'); staleCache.stop(); }); tap.test('cache: should reject content past stale deadline', async () => { const veryShortCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 1, staleWhileRevalidate: true, staleMaxAgeSeconds: 1, // Only 1 second stale window immutableTtlSeconds: 1, negativeCacheTtlSeconds: 1, }, 1000, storageBackend ); const context = createFetchContext({ resource: 'very-stale-test' }); await veryShortCache.set( context, Buffer.from('will expire completely'), 'text/plain', {}, 'short-upstream', 'https://short.example.com' ); // Wait for both fresh AND stale periods to expire await new Promise(resolve => setTimeout(resolve, 2500)); const entry = await veryShortCache.get(context, 'https://short.example.com'); expect(entry).toBeNull(); veryShortCache.stop(); }); // ============================================================================ // Negative Cache Tests // ============================================================================ tap.test('cache: should store negative cache entries (404)', async () => { const context = createFetchContext({ resource: 'not-found-pkg' }); const upstreamUrl = 'https://registry.npmjs.org'; await cache.setNegative(context, 'npmjs', upstreamUrl); const hasNegative = await cache.hasNegative(context, upstreamUrl); expect(hasNegative).toBeTrue(); }); tap.test('cache: should retrieve negative cache entries', async () => { const context = createFetchContext({ resource: 'negative-retrieve-test' }); const upstreamUrl = 'https://registry.npmjs.org'; await cache.setNegative(context, 'npmjs', upstreamUrl); const entry = await cache.get(context, upstreamUrl); expect(entry).toBeTruthy(); expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404 }); // ============================================================================ // Eviction Tests // ============================================================================ tap.test('cache: should evict oldest entries when memory limit reached', async () => { // Create cache with very small limit const smallCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 300 }, 5, // Only 5 entries storageBackend ); // Add 10 entries for (let i = 0; i < 10; i++) { const context = createFetchContext({ resource: `evict-test-${i}` }); await smallCache.set( context, Buffer.from(`data ${i}`), 'text/plain', {}, 'evict-upstream', 'https://evict.example.com' ); } const stats = smallCache.getStats(); // Should have evicted some entries expect(stats.totalEntries).toBeLessThanOrEqual(5); smallCache.stop(); }); // ============================================================================ // Query Parameter Tests // ============================================================================ tap.test('cache: cache key should include query parameters', async () => { const context1 = createFetchContext({ resource: 'query-test', query: { version: '1.0.0' }, }); const context2 = createFetchContext({ resource: 'query-test', query: { version: '2.0.0' }, }); const upstreamUrl = 'https://registry.npmjs.org'; // Store with v1 query await cache.set( context1, Buffer.from('version 1 data'), 'text/plain', {}, 'query-upstream', upstreamUrl ); // Store with v2 query await cache.set( context2, Buffer.from('version 2 data'), 'text/plain', {}, 'query-upstream', upstreamUrl ); // Retrieve v1 - should get v1 data const entry1 = await cache.get(context1, upstreamUrl); expect(entry1).toBeTruthy(); expect(entry1!.data.toString()).toEqual('version 1 data'); // Retrieve v2 - should get v2 data const entry2 = await cache.get(context2, upstreamUrl); expect(entry2).toBeTruthy(); expect(entry2!.data.toString()).toEqual('version 2 data'); }); // ============================================================================ // S3 Persistence Tests // ============================================================================ tap.test('cache: should load from S3 on memory cache miss', async () => { // Use a unique resource name for this test const uniqueResource = `persist-test-${Date.now()}`; const persistContext = createFetchContext({ resource: uniqueResource }); const upstreamUrl = 'https://persist.example.com'; // Store in first cache instance await cache.set( persistContext, Buffer.from('persisted data'), 'text/plain', {}, 'persist-upstream', upstreamUrl ); // Wait for S3 write to complete await new Promise(resolve => setTimeout(resolve, 200)); // Verify the entry is in the original cache's memory const originalEntry = await cache.get(persistContext, upstreamUrl); expect(originalEntry).toBeTruthy(); // Create a new cache instance (simulates restart) with SAME storage backend const freshCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 300 }, 10000, storageBackend ); // Fresh cache has empty memory, should load from S3 const entry = await freshCache.get(persistContext, upstreamUrl); expect(entry).toBeTruthy(); expect(entry!.data.toString()).toEqual('persisted data'); freshCache.stop(); }); // ============================================================================ // Cache Stats Tests // ============================================================================ tap.test('cache: should return accurate stats', async () => { const statsCache = new UpstreamCache( { enabled: true, defaultTtlSeconds: 300 }, 1000, storageBackend ); // Add some entries for (let i = 0; i < 3; i++) { const context = createFetchContext({ resource: `stats-test-${i}` }); await statsCache.set( context, Buffer.from(`stats data ${i}`), 'text/plain', {}, 'stats-upstream', 'https://stats.example.com' ); } // Add a negative entry const negContext = createFetchContext({ resource: 'stats-negative' }); await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com'); const stats = statsCache.getStats(); expect(stats.totalEntries).toBeGreaterThanOrEqual(4); expect(stats.enabled).toBeTrue(); expect(stats.hasStorage).toBeTrue(); expect(stats.maxEntries).toEqual(1000); statsCache.stop(); }); // ============================================================================ // Invalidation Tests // ============================================================================ tap.test('cache: should invalidate specific cache entry', async () => { const invalidateContext = createFetchContext({ resource: 'invalidate-test' }); const upstreamUrl = 'https://invalidate.example.com'; await cache.set( invalidateContext, Buffer.from('to be invalidated'), 'text/plain', {}, 'inv-upstream', upstreamUrl ); // Verify it exists let entry = await cache.get(invalidateContext, upstreamUrl); expect(entry).toBeTruthy(); // Invalidate const deleted = await cache.invalidate(invalidateContext, upstreamUrl); expect(deleted).toBeTrue(); // Should be gone entry = await cache.get(invalidateContext, upstreamUrl); expect(entry).toBeNull(); }); tap.test('cache: should invalidate entries matching pattern', async () => { const upstreamUrl = 'https://pattern.example.com'; // Add multiple entries for (const name of ['pattern-a', 'pattern-b', 'other-c']) { const context = createFetchContext({ resource: name }); await cache.set( context, Buffer.from(`data for ${name}`), 'text/plain', {}, 'pattern-upstream', upstreamUrl ); } // Invalidate entries matching 'pattern-*' const count = await cache.invalidatePattern(/pattern-/); expect(count).toBeGreaterThanOrEqual(2); // pattern-a should be gone const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl); expect(entryA).toBeNull(); // other-c should still exist const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl); expect(entryC).toBeTruthy(); }); // ============================================================================ // Cleanup // ============================================================================ tap.test('cleanup: should stop cache and clean up bucket', async () => { if (cache) { cache.stop(); } // Clean up test bucket if (s3Bucket) { try { const files = await s3Bucket.fastList({}); for (const file of files) { await s3Bucket.fastRemove({ path: file.name }); } await smartBucket.removeBucket(bucketName); } catch { // Ignore cleanup errors } } }); export default tap.start();