feat(core): Add core registry infrastructure: storage, auth, upstream cache, and protocol handlers
This commit is contained in:
598
test/test.upstream.cache.s3.ts
Normal file
598
test/test.upstream.cache.s3.ts
Normal file
@@ -0,0 +1,598 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js';
|
||||
import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js';
|
||||
import type { IStorageBackend } from '../ts/core/interfaces.core.js';
|
||||
import { generateTestRunId } from './helpers/registry.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit');
|
||||
|
||||
// ============================================================================
|
||||
// Test State
|
||||
// ============================================================================
|
||||
|
||||
let cache: UpstreamCache;
|
||||
let storageBackend: IStorageBackend;
|
||||
let s3Bucket: smartbucket.Bucket;
|
||||
let smartBucket: smartbucket.SmartBucket;
|
||||
let testRunId: string;
|
||||
let bucketName: string;
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
function createFetchContext(overrides?: Partial<IUpstreamFetchContext>): IUpstreamFetchContext {
|
||||
// Use resource name as path to ensure unique cache keys
|
||||
const resource = overrides?.resource || 'lodash';
|
||||
return {
|
||||
protocol: 'npm',
|
||||
resource,
|
||||
resourceType: 'packument',
|
||||
path: `/${resource}`,
|
||||
method: 'GET',
|
||||
headers: {},
|
||||
query: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Setup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('setup: should create S3 storage backend', async () => {
|
||||
testRunId = generateTestRunId();
|
||||
bucketName = `test-ucache-${testRunId.substring(0, 8)}`;
|
||||
|
||||
const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
|
||||
const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
|
||||
const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
|
||||
const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');
|
||||
|
||||
smartBucket = new smartbucket.SmartBucket({
|
||||
accessKey: s3AccessKey || 'minioadmin',
|
||||
accessSecret: s3SecretKey || 'minioadmin',
|
||||
endpoint: s3Endpoint || 'localhost',
|
||||
port: parseInt(s3Port || '9000', 10),
|
||||
useSsl: false,
|
||||
});
|
||||
|
||||
s3Bucket = await smartBucket.createBucket(bucketName);
|
||||
|
||||
// Create storage backend adapter
|
||||
storageBackend = {
|
||||
getObject: async (key: string): Promise<Buffer | null> => {
|
||||
try {
|
||||
// fastGet returns Buffer directly (or undefined if not found)
|
||||
const data = await s3Bucket.fastGet({ path: key });
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
return data;
|
||||
} catch (error) {
|
||||
// fastGet throws if object doesn't exist
|
||||
return null;
|
||||
}
|
||||
},
|
||||
putObject: async (key: string, data: Buffer): Promise<void> => {
|
||||
await s3Bucket.fastPut({ path: key, contents: data, overwrite: true });
|
||||
},
|
||||
deleteObject: async (key: string): Promise<void> => {
|
||||
await s3Bucket.fastRemove({ path: key });
|
||||
},
|
||||
listObjects: async (prefix: string): Promise<string[]> => {
|
||||
const paths: string[] = [];
|
||||
for await (const path of s3Bucket.listAllObjects(prefix)) {
|
||||
paths.push(path);
|
||||
}
|
||||
return paths;
|
||||
},
|
||||
};
|
||||
|
||||
expect(storageBackend).toBeTruthy();
|
||||
});
|
||||
|
||||
tap.test('setup: should create UpstreamCache with S3 storage', async () => {
|
||||
cache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
10000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
expect(cache.isEnabled()).toBeTrue();
|
||||
expect(cache.hasStorage()).toBeTrue();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Basic Cache Operations
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should store cache entry in S3', async () => {
|
||||
const context = createFetchContext({ resource: 'store-test' });
|
||||
const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' }));
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'application/json',
|
||||
{ 'etag': '"abc123"' },
|
||||
'npmjs',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Verify in S3
|
||||
const stats = cache.getStats();
|
||||
expect(stats.totalEntries).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
tap.test('cache: should retrieve cache entry from S3', async () => {
|
||||
const context = createFetchContext({ resource: 'retrieve-test' });
|
||||
const testData = Buffer.from('retrieve test data');
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'application/octet-stream',
|
||||
{},
|
||||
'npmjs',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
const entry = await cache.get(context, upstreamUrl);
|
||||
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('retrieve test data');
|
||||
expect(entry!.contentType).toEqual('application/octet-stream');
|
||||
expect(entry!.upstreamId).toEqual('npmjs');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Multi-Upstream URL Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should include upstream URL in cache path', async () => {
|
||||
const context = createFetchContext({ resource: 'url-path-test' });
|
||||
const testData = Buffer.from('url path test');
|
||||
|
||||
await cache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'npmjs',
|
||||
'https://registry.npmjs.org'
|
||||
);
|
||||
|
||||
// The cache key should include the escaped URL
|
||||
const entry = await cache.get(context, 'https://registry.npmjs.org');
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('url path test');
|
||||
});
|
||||
|
||||
tap.test('cache: should handle multiple upstreams with different URLs', async () => {
|
||||
const context = createFetchContext({ resource: '@company/private-pkg' });
|
||||
|
||||
// Store from private upstream
|
||||
const privateData = Buffer.from('private package data');
|
||||
await cache.set(
|
||||
context,
|
||||
privateData,
|
||||
'application/json',
|
||||
{},
|
||||
'private-npm',
|
||||
'https://npm.company.com'
|
||||
);
|
||||
|
||||
// Store from public upstream (same resource name, different upstream)
|
||||
const publicData = Buffer.from('public package data');
|
||||
await cache.set(
|
||||
context,
|
||||
publicData,
|
||||
'application/json',
|
||||
{},
|
||||
'public-npm',
|
||||
'https://registry.npmjs.org'
|
||||
);
|
||||
|
||||
// Retrieve from private - should get private data
|
||||
const privateEntry = await cache.get(context, 'https://npm.company.com');
|
||||
expect(privateEntry).toBeTruthy();
|
||||
expect(privateEntry!.data.toString()).toEqual('private package data');
|
||||
expect(privateEntry!.upstreamId).toEqual('private-npm');
|
||||
|
||||
// Retrieve from public - should get public data
|
||||
const publicEntry = await cache.get(context, 'https://registry.npmjs.org');
|
||||
expect(publicEntry).toBeTruthy();
|
||||
expect(publicEntry!.data.toString()).toEqual('public package data');
|
||||
expect(publicEntry!.upstreamId).toEqual('public-npm');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// TTL and Expiration Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should respect TTL expiration', async () => {
|
||||
// Create cache with very short TTL
|
||||
const shortTtlCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1, // 1 second TTL
|
||||
staleWhileRevalidate: false,
|
||||
staleMaxAgeSeconds: 0,
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'ttl-test' });
|
||||
const testData = Buffer.from('expires soon');
|
||||
|
||||
await shortTtlCache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'test-upstream',
|
||||
'https://test.example.com'
|
||||
);
|
||||
|
||||
// Should exist immediately
|
||||
let entry = await shortTtlCache.get(context, 'https://test.example.com');
|
||||
expect(entry).toBeTruthy();
|
||||
|
||||
// Wait for expiration
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
// Should be expired now
|
||||
entry = await shortTtlCache.get(context, 'https://test.example.com');
|
||||
expect(entry).toBeNull();
|
||||
|
||||
shortTtlCache.stop();
|
||||
});
|
||||
|
||||
tap.test('cache: should serve stale content during stale-while-revalidate window', async () => {
|
||||
const staleCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1, // 1 second fresh
|
||||
staleWhileRevalidate: true,
|
||||
staleMaxAgeSeconds: 60, // 60 seconds stale window
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'stale-test' });
|
||||
const testData = Buffer.from('stale but usable');
|
||||
|
||||
await staleCache.set(
|
||||
context,
|
||||
testData,
|
||||
'text/plain',
|
||||
{},
|
||||
'stale-upstream',
|
||||
'https://stale.example.com'
|
||||
);
|
||||
|
||||
// Wait for fresh period to expire
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
// Should still be available but marked as stale
|
||||
const entry = await staleCache.get(context, 'https://stale.example.com');
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.stale).toBeTrue();
|
||||
expect(entry!.data.toString()).toEqual('stale but usable');
|
||||
|
||||
staleCache.stop();
|
||||
});
|
||||
|
||||
tap.test('cache: should reject content past stale deadline', async () => {
|
||||
const veryShortCache = new UpstreamCache(
|
||||
{
|
||||
enabled: true,
|
||||
defaultTtlSeconds: 1,
|
||||
staleWhileRevalidate: true,
|
||||
staleMaxAgeSeconds: 1, // Only 1 second stale window
|
||||
immutableTtlSeconds: 1,
|
||||
negativeCacheTtlSeconds: 1,
|
||||
},
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
const context = createFetchContext({ resource: 'very-stale-test' });
|
||||
await veryShortCache.set(
|
||||
context,
|
||||
Buffer.from('will expire completely'),
|
||||
'text/plain',
|
||||
{},
|
||||
'short-upstream',
|
||||
'https://short.example.com'
|
||||
);
|
||||
|
||||
// Wait for both fresh AND stale periods to expire
|
||||
await new Promise(resolve => setTimeout(resolve, 2500));
|
||||
|
||||
const entry = await veryShortCache.get(context, 'https://short.example.com');
|
||||
expect(entry).toBeNull();
|
||||
|
||||
veryShortCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Negative Cache Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should store negative cache entries (404)', async () => {
|
||||
const context = createFetchContext({ resource: 'not-found-pkg' });
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.setNegative(context, 'npmjs', upstreamUrl);
|
||||
|
||||
const hasNegative = await cache.hasNegative(context, upstreamUrl);
|
||||
expect(hasNegative).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('cache: should retrieve negative cache entries', async () => {
|
||||
const context = createFetchContext({ resource: 'negative-retrieve-test' });
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
await cache.setNegative(context, 'npmjs', upstreamUrl);
|
||||
|
||||
const entry = await cache.get(context, upstreamUrl);
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Eviction Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should evict oldest entries when memory limit reached', async () => {
|
||||
// Create cache with very small limit
|
||||
const smallCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
5, // Only 5 entries
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Add 10 entries
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const context = createFetchContext({ resource: `evict-test-${i}` });
|
||||
await smallCache.set(
|
||||
context,
|
||||
Buffer.from(`data ${i}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'evict-upstream',
|
||||
'https://evict.example.com'
|
||||
);
|
||||
}
|
||||
|
||||
const stats = smallCache.getStats();
|
||||
// Should have evicted some entries
|
||||
expect(stats.totalEntries).toBeLessThanOrEqual(5);
|
||||
|
||||
smallCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Query Parameter Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: cache key should include query parameters', async () => {
|
||||
const context1 = createFetchContext({
|
||||
resource: 'query-test',
|
||||
query: { version: '1.0.0' },
|
||||
});
|
||||
|
||||
const context2 = createFetchContext({
|
||||
resource: 'query-test',
|
||||
query: { version: '2.0.0' },
|
||||
});
|
||||
|
||||
const upstreamUrl = 'https://registry.npmjs.org';
|
||||
|
||||
// Store with v1 query
|
||||
await cache.set(
|
||||
context1,
|
||||
Buffer.from('version 1 data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'query-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Store with v2 query
|
||||
await cache.set(
|
||||
context2,
|
||||
Buffer.from('version 2 data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'query-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Retrieve v1 - should get v1 data
|
||||
const entry1 = await cache.get(context1, upstreamUrl);
|
||||
expect(entry1).toBeTruthy();
|
||||
expect(entry1!.data.toString()).toEqual('version 1 data');
|
||||
|
||||
// Retrieve v2 - should get v2 data
|
||||
const entry2 = await cache.get(context2, upstreamUrl);
|
||||
expect(entry2).toBeTruthy();
|
||||
expect(entry2!.data.toString()).toEqual('version 2 data');
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// S3 Persistence Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should load from S3 on memory cache miss', async () => {
|
||||
// Use a unique resource name for this test
|
||||
const uniqueResource = `persist-test-${Date.now()}`;
|
||||
const persistContext = createFetchContext({ resource: uniqueResource });
|
||||
const upstreamUrl = 'https://persist.example.com';
|
||||
|
||||
// Store in first cache instance
|
||||
await cache.set(
|
||||
persistContext,
|
||||
Buffer.from('persisted data'),
|
||||
'text/plain',
|
||||
{},
|
||||
'persist-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Wait for S3 write to complete
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
// Verify the entry is in the original cache's memory
|
||||
const originalEntry = await cache.get(persistContext, upstreamUrl);
|
||||
expect(originalEntry).toBeTruthy();
|
||||
|
||||
// Create a new cache instance (simulates restart) with SAME storage backend
|
||||
const freshCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
10000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Fresh cache has empty memory, should load from S3
|
||||
const entry = await freshCache.get(persistContext, upstreamUrl);
|
||||
|
||||
expect(entry).toBeTruthy();
|
||||
expect(entry!.data.toString()).toEqual('persisted data');
|
||||
|
||||
freshCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cache Stats Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should return accurate stats', async () => {
|
||||
const statsCache = new UpstreamCache(
|
||||
{ enabled: true, defaultTtlSeconds: 300 },
|
||||
1000,
|
||||
storageBackend
|
||||
);
|
||||
|
||||
// Add some entries
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const context = createFetchContext({ resource: `stats-test-${i}` });
|
||||
await statsCache.set(
|
||||
context,
|
||||
Buffer.from(`stats data ${i}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'stats-upstream',
|
||||
'https://stats.example.com'
|
||||
);
|
||||
}
|
||||
|
||||
// Add a negative entry
|
||||
const negContext = createFetchContext({ resource: 'stats-negative' });
|
||||
await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com');
|
||||
|
||||
const stats = statsCache.getStats();
|
||||
|
||||
expect(stats.totalEntries).toBeGreaterThanOrEqual(4);
|
||||
expect(stats.enabled).toBeTrue();
|
||||
expect(stats.hasStorage).toBeTrue();
|
||||
expect(stats.maxEntries).toEqual(1000);
|
||||
|
||||
statsCache.stop();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Invalidation Tests
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cache: should invalidate specific cache entry', async () => {
|
||||
const invalidateContext = createFetchContext({ resource: 'invalidate-test' });
|
||||
const upstreamUrl = 'https://invalidate.example.com';
|
||||
|
||||
await cache.set(
|
||||
invalidateContext,
|
||||
Buffer.from('to be invalidated'),
|
||||
'text/plain',
|
||||
{},
|
||||
'inv-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
|
||||
// Verify it exists
|
||||
let entry = await cache.get(invalidateContext, upstreamUrl);
|
||||
expect(entry).toBeTruthy();
|
||||
|
||||
// Invalidate
|
||||
const deleted = await cache.invalidate(invalidateContext, upstreamUrl);
|
||||
expect(deleted).toBeTrue();
|
||||
|
||||
// Should be gone
|
||||
entry = await cache.get(invalidateContext, upstreamUrl);
|
||||
expect(entry).toBeNull();
|
||||
});
|
||||
|
||||
tap.test('cache: should invalidate entries matching pattern', async () => {
|
||||
const upstreamUrl = 'https://pattern.example.com';
|
||||
|
||||
// Add multiple entries
|
||||
for (const name of ['pattern-a', 'pattern-b', 'other-c']) {
|
||||
const context = createFetchContext({ resource: name });
|
||||
await cache.set(
|
||||
context,
|
||||
Buffer.from(`data for ${name}`),
|
||||
'text/plain',
|
||||
{},
|
||||
'pattern-upstream',
|
||||
upstreamUrl
|
||||
);
|
||||
}
|
||||
|
||||
// Invalidate entries matching 'pattern-*'
|
||||
const count = await cache.invalidatePattern(/pattern-/);
|
||||
expect(count).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// pattern-a should be gone
|
||||
const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl);
|
||||
expect(entryA).toBeNull();
|
||||
|
||||
// other-c should still exist
|
||||
const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl);
|
||||
expect(entryC).toBeTruthy();
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Cleanup
|
||||
// ============================================================================
|
||||
|
||||
tap.test('cleanup: should stop cache and clean up bucket', async () => {
|
||||
if (cache) {
|
||||
cache.stop();
|
||||
}
|
||||
|
||||
// Clean up test bucket
|
||||
if (s3Bucket) {
|
||||
try {
|
||||
const files = await s3Bucket.fastList({});
|
||||
for (const file of files) {
|
||||
await s3Bucket.fastRemove({ path: file.name });
|
||||
}
|
||||
await smartBucket.removeBucket(bucketName);
|
||||
} catch {
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user