smartregistry/test/test.upstream.cache.s3.ts

import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartbucket from '@push.rocks/smartbucket';
import { UpstreamCache } from '../ts/upstream/classes.upstreamcache.js';
import type { IUpstreamFetchContext, IUpstreamCacheConfig } from '../ts/upstream/interfaces.upstream.js';
import type { IStorageBackend } from '../ts/core/interfaces.core.js';
import { generateTestRunId } from './helpers/registry.js';

const testQenv = new qenv.Qenv('./', './.nogit');

// ============================================================================
// Test State
// ============================================================================

let cache: UpstreamCache;
let storageBackend: IStorageBackend;
let s3Bucket: smartbucket.Bucket;
let smartBucket: smartbucket.SmartBucket;
let testRunId: string;
let bucketName: string;

// ============================================================================
// Helper Functions
// ============================================================================

function createFetchContext(overrides?: Partial<IUpstreamFetchContext>): IUpstreamFetchContext {
  // Use resource name as path to ensure unique cache keys
  const resource = overrides?.resource || 'lodash';
  return {
    protocol: 'npm',
    resource,
    resourceType: 'packument',
    path: `/${resource}`,
    method: 'GET',
    headers: {},
    query: {},
    ...overrides,
  };
}

// ============================================================================
// Setup
// ============================================================================

tap.test('setup: should create S3 storage backend', async () => {
  testRunId = generateTestRunId();
  bucketName = `test-ucache-${testRunId.substring(0, 8)}`;

  const s3AccessKey = await testQenv.getEnvVarOnDemand('S3_ACCESSKEY');
  const s3SecretKey = await testQenv.getEnvVarOnDemand('S3_SECRETKEY');
  const s3Endpoint = await testQenv.getEnvVarOnDemand('S3_ENDPOINT');
  const s3Port = await testQenv.getEnvVarOnDemand('S3_PORT');

  smartBucket = new smartbucket.SmartBucket({
    accessKey: s3AccessKey || 'minioadmin',
    accessSecret: s3SecretKey || 'minioadmin',
    endpoint: s3Endpoint || 'localhost',
    port: parseInt(s3Port || '9000', 10),
    useSsl: false,
  });

  s3Bucket = await smartBucket.createBucket(bucketName);

  // Create storage backend adapter
  storageBackend = {
    getObject: async (key: string): Promise<Buffer | null> => {
      try {
        // fastGet returns Buffer directly (or undefined if not found)
        const data = await s3Bucket.fastGet({ path: key });
        if (!data) {
          return null;
        }
        return data;
      } catch (error) {
        // fastGet throws if object doesn't exist
        return null;
      }
    },
    putObject: async (key: string, data: Buffer): Promise<void> => {
      await s3Bucket.fastPut({ path: key, contents: data, overwrite: true });
    },
    deleteObject: async (key: string): Promise<void> => {
      await s3Bucket.fastRemove({ path: key });
    },
    listObjects: async (prefix: string): Promise<string[]> => {
      const paths: string[] = [];
      for await (const path of s3Bucket.listAllObjects(prefix)) {
        paths.push(path);
      }
      return paths;
    },
  };

  expect(storageBackend).toBeTruthy();
});

tap.test('setup: should create UpstreamCache with S3 storage', async () => {
  cache = new UpstreamCache(
    { enabled: true, defaultTtlSeconds: 300 },
    10000,
    storageBackend
  );

  expect(cache.isEnabled()).toBeTrue();
  expect(cache.hasStorage()).toBeTrue();
});

// ============================================================================
// Basic Cache Operations
// ============================================================================

tap.test('cache: should store cache entry in S3', async () => {
  const context = createFetchContext({ resource: 'store-test' });
  const testData = Buffer.from(JSON.stringify({ name: 'store-test', version: '1.0.0' }));
  const upstreamUrl = 'https://registry.npmjs.org';

  await cache.set(
    context,
    testData,
    'application/json',
    { 'etag': '"abc123"' },
    'npmjs',
    upstreamUrl
  );

  // Verify in S3
  const stats = cache.getStats();
  expect(stats.totalEntries).toBeGreaterThanOrEqual(1);
});

tap.test('cache: should retrieve cache entry from S3', async () => {
  const context = createFetchContext({ resource: 'retrieve-test' });
  const testData = Buffer.from('retrieve test data');
  const upstreamUrl = 'https://registry.npmjs.org';

  await cache.set(
    context,
    testData,
    'application/octet-stream',
    {},
    'npmjs',
    upstreamUrl
  );

  const entry = await cache.get(context, upstreamUrl);

  expect(entry).toBeTruthy();
  expect(entry!.data.toString()).toEqual('retrieve test data');
  expect(entry!.contentType).toEqual('application/octet-stream');
  expect(entry!.upstreamId).toEqual('npmjs');
});

// ============================================================================
// Multi-Upstream URL Tests
// ============================================================================

tap.test('cache: should include upstream URL in cache path', async () => {
  const context = createFetchContext({ resource: 'url-path-test' });
  const testData = Buffer.from('url path test');

  await cache.set(
    context,
    testData,
    'text/plain',
    {},
    'npmjs',
    'https://registry.npmjs.org'
  );

  // The cache key should include the escaped URL
  const entry = await cache.get(context, 'https://registry.npmjs.org');
  expect(entry).toBeTruthy();
  expect(entry!.data.toString()).toEqual('url path test');
});

tap.test('cache: should handle multiple upstreams with different URLs', async () => {
  const context = createFetchContext({ resource: '@company/private-pkg' });

  // Store from private upstream
  const privateData = Buffer.from('private package data');
  await cache.set(
    context,
    privateData,
    'application/json',
    {},
    'private-npm',
    'https://npm.company.com'
  );

  // Store from public upstream (same resource name, different upstream)
  const publicData = Buffer.from('public package data');
  await cache.set(
    context,
    publicData,
    'application/json',
    {},
    'public-npm',
    'https://registry.npmjs.org'
  );

  // Retrieve from private - should get private data
  const privateEntry = await cache.get(context, 'https://npm.company.com');
  expect(privateEntry).toBeTruthy();
  expect(privateEntry!.data.toString()).toEqual('private package data');
  expect(privateEntry!.upstreamId).toEqual('private-npm');

  // Retrieve from public - should get public data
  const publicEntry = await cache.get(context, 'https://registry.npmjs.org');
  expect(publicEntry).toBeTruthy();
  expect(publicEntry!.data.toString()).toEqual('public package data');
  expect(publicEntry!.upstreamId).toEqual('public-npm');
});

// ============================================================================
// TTL and Expiration Tests
// ============================================================================

tap.test('cache: should respect TTL expiration', async () => {
  // Create cache with very short TTL
  const shortTtlCache = new UpstreamCache(
    {
      enabled: true,
      defaultTtlSeconds: 1, // 1 second TTL
      staleWhileRevalidate: false,
      staleMaxAgeSeconds: 0,
      immutableTtlSeconds: 1,
      negativeCacheTtlSeconds: 1,
    },
    1000,
    storageBackend
  );

  const context = createFetchContext({ resource: 'ttl-test' });
  const testData = Buffer.from('expires soon');

  await shortTtlCache.set(
    context,
    testData,
    'text/plain',
    {},
    'test-upstream',
    'https://test.example.com'
  );

  // Should exist immediately
  let entry = await shortTtlCache.get(context, 'https://test.example.com');
  expect(entry).toBeTruthy();

  // Wait for expiration
  await new Promise(resolve => setTimeout(resolve, 1500));

  // Should be expired now
  entry = await shortTtlCache.get(context, 'https://test.example.com');
  expect(entry).toBeNull();

  shortTtlCache.stop();
});

tap.test('cache: should serve stale content during stale-while-revalidate window', async () => {
  const staleCache = new UpstreamCache(
    {
      enabled: true,
      defaultTtlSeconds: 1, // 1 second fresh
      staleWhileRevalidate: true,
      staleMaxAgeSeconds: 60, // 60 seconds stale window
      immutableTtlSeconds: 1,
      negativeCacheTtlSeconds: 1,
    },
    1000,
    storageBackend
  );

  const context = createFetchContext({ resource: 'stale-test' });
  const testData = Buffer.from('stale but usable');

  await staleCache.set(
    context,
    testData,
    'text/plain',
    {},
    'stale-upstream',
    'https://stale.example.com'
  );

  // Wait for fresh period to expire
  await new Promise(resolve => setTimeout(resolve, 1500));

  // Should still be available but marked as stale
  const entry = await staleCache.get(context, 'https://stale.example.com');
  expect(entry).toBeTruthy();
  expect(entry!.stale).toBeTrue();
  expect(entry!.data.toString()).toEqual('stale but usable');

  staleCache.stop();
});

tap.test('cache: should reject content past stale deadline', async () => {
  const veryShortCache = new UpstreamCache(
    {
      enabled: true,
      defaultTtlSeconds: 1,
      staleWhileRevalidate: true,
      staleMaxAgeSeconds: 1, // Only 1 second stale window
      immutableTtlSeconds: 1,
      negativeCacheTtlSeconds: 1,
    },
    1000,
    storageBackend
  );

  const context = createFetchContext({ resource: 'very-stale-test' });
  await veryShortCache.set(
    context,
    Buffer.from('will expire completely'),
    'text/plain',
    {},
    'short-upstream',
    'https://short.example.com'
  );

  // Wait for both fresh AND stale periods to expire
  await new Promise(resolve => setTimeout(resolve, 2500));

  const entry = await veryShortCache.get(context, 'https://short.example.com');
  expect(entry).toBeNull();

  veryShortCache.stop();
});

// ============================================================================
// Negative Cache Tests
// ============================================================================

tap.test('cache: should store negative cache entries (404)', async () => {
  const context = createFetchContext({ resource: 'not-found-pkg' });
  const upstreamUrl = 'https://registry.npmjs.org';

  await cache.setNegative(context, 'npmjs', upstreamUrl);

  const hasNegative = await cache.hasNegative(context, upstreamUrl);
  expect(hasNegative).toBeTrue();
});

tap.test('cache: should retrieve negative cache entries', async () => {
  const context = createFetchContext({ resource: 'negative-retrieve-test' });
  const upstreamUrl = 'https://registry.npmjs.org';

  await cache.setNegative(context, 'npmjs', upstreamUrl);

  const entry = await cache.get(context, upstreamUrl);
  expect(entry).toBeTruthy();
  expect(entry!.data.length).toEqual(0); // Empty buffer indicates 404
});

// ============================================================================
// Eviction Tests
// ============================================================================

tap.test('cache: should evict oldest entries when memory limit reached', async () => {
  // Create cache with very small limit
  const smallCache = new UpstreamCache(
    { enabled: true, defaultTtlSeconds: 300 },
    5, // Only 5 entries
    storageBackend
  );

  // Add 10 entries
  for (let i = 0; i < 10; i++) {
    const context = createFetchContext({ resource: `evict-test-${i}` });
    await smallCache.set(
      context,
      Buffer.from(`data ${i}`),
      'text/plain',
      {},
      'evict-upstream',
      'https://evict.example.com'
    );
  }

  const stats = smallCache.getStats();
  // Should have evicted some entries
  expect(stats.totalEntries).toBeLessThanOrEqual(5);

  smallCache.stop();
});

// ============================================================================
// Query Parameter Tests
// ============================================================================

tap.test('cache: cache key should include query parameters', async () => {
  const context1 = createFetchContext({
    resource: 'query-test',
    query: { version: '1.0.0' },
  });

  const context2 = createFetchContext({
    resource: 'query-test',
    query: { version: '2.0.0' },
  });

  const upstreamUrl = 'https://registry.npmjs.org';

  // Store with v1 query
  await cache.set(
    context1,
    Buffer.from('version 1 data'),
    'text/plain',
    {},
    'query-upstream',
    upstreamUrl
  );

  // Store with v2 query
  await cache.set(
    context2,
    Buffer.from('version 2 data'),
    'text/plain',
    {},
    'query-upstream',
    upstreamUrl
  );

  // Retrieve v1 - should get v1 data
  const entry1 = await cache.get(context1, upstreamUrl);
  expect(entry1).toBeTruthy();
  expect(entry1!.data.toString()).toEqual('version 1 data');

  // Retrieve v2 - should get v2 data
  const entry2 = await cache.get(context2, upstreamUrl);
  expect(entry2).toBeTruthy();
  expect(entry2!.data.toString()).toEqual('version 2 data');
});

// ============================================================================
// S3 Persistence Tests
// ============================================================================

tap.test('cache: should load from S3 on memory cache miss', async () => {
  // Use a unique resource name for this test
  const uniqueResource = `persist-test-${Date.now()}`;
  const persistContext = createFetchContext({ resource: uniqueResource });
  const upstreamUrl = 'https://persist.example.com';

  // Store in first cache instance
  await cache.set(
    persistContext,
    Buffer.from('persisted data'),
    'text/plain',
    {},
    'persist-upstream',
    upstreamUrl
  );

  // Wait for S3 write to complete
  await new Promise(resolve => setTimeout(resolve, 200));

  // Verify the entry is in the original cache's memory
  const originalEntry = await cache.get(persistContext, upstreamUrl);
  expect(originalEntry).toBeTruthy();

  // Create a new cache instance (simulates restart) with SAME storage backend
  const freshCache = new UpstreamCache(
    { enabled: true, defaultTtlSeconds: 300 },
    10000,
    storageBackend
  );

  // Fresh cache has empty memory, should load from S3
  const entry = await freshCache.get(persistContext, upstreamUrl);

  expect(entry).toBeTruthy();
  expect(entry!.data.toString()).toEqual('persisted data');

  freshCache.stop();
});

// ============================================================================
// Cache Stats Tests
// ============================================================================

tap.test('cache: should return accurate stats', async () => {
  const statsCache = new UpstreamCache(
    { enabled: true, defaultTtlSeconds: 300 },
    1000,
    storageBackend
  );

  // Add some entries
  for (let i = 0; i < 3; i++) {
    const context = createFetchContext({ resource: `stats-test-${i}` });
    await statsCache.set(
      context,
      Buffer.from(`stats data ${i}`),
      'text/plain',
      {},
      'stats-upstream',
      'https://stats.example.com'
    );
  }

  // Add a negative entry
  const negContext = createFetchContext({ resource: 'stats-negative' });
  await statsCache.setNegative(negContext, 'stats-upstream', 'https://stats.example.com');

  const stats = statsCache.getStats();

  expect(stats.totalEntries).toBeGreaterThanOrEqual(4);
  expect(stats.enabled).toBeTrue();
  expect(stats.hasStorage).toBeTrue();
  expect(stats.maxEntries).toEqual(1000);

  statsCache.stop();
});

// ============================================================================
// Invalidation Tests
// ============================================================================

tap.test('cache: should invalidate specific cache entry', async () => {
  const invalidateContext = createFetchContext({ resource: 'invalidate-test' });
  const upstreamUrl = 'https://invalidate.example.com';

  await cache.set(
    invalidateContext,
    Buffer.from('to be invalidated'),
    'text/plain',
    {},
    'inv-upstream',
    upstreamUrl
  );

  // Verify it exists
  let entry = await cache.get(invalidateContext, upstreamUrl);
  expect(entry).toBeTruthy();

  // Invalidate
  const deleted = await cache.invalidate(invalidateContext, upstreamUrl);
  expect(deleted).toBeTrue();

  // Should be gone
  entry = await cache.get(invalidateContext, upstreamUrl);
  expect(entry).toBeNull();
});

tap.test('cache: should invalidate entries matching pattern', async () => {
  const upstreamUrl = 'https://pattern.example.com';

  // Add multiple entries
  for (const name of ['pattern-a', 'pattern-b', 'other-c']) {
    const context = createFetchContext({ resource: name });
    await cache.set(
      context,
      Buffer.from(`data for ${name}`),
      'text/plain',
      {},
      'pattern-upstream',
      upstreamUrl
    );
  }

  // Invalidate entries matching 'pattern-*'
  const count = await cache.invalidatePattern(/pattern-/);
  expect(count).toBeGreaterThanOrEqual(2);

  // pattern-a should be gone
  const entryA = await cache.get(createFetchContext({ resource: 'pattern-a' }), upstreamUrl);
  expect(entryA).toBeNull();

  // other-c should still exist
  const entryC = await cache.get(createFetchContext({ resource: 'other-c' }), upstreamUrl);
  expect(entryC).toBeTruthy();
});

// ============================================================================
// Cleanup
// ============================================================================

tap.test('cleanup: should stop cache and clean up bucket', async () => {
  if (cache) {
    cache.stop();
  }

  // Clean up test bucket
  if (s3Bucket) {
    try {
      const files = await s3Bucket.fastList({});
      for (const file of files) {
        await s3Bucket.fastRemove({ path: file.name });
      }
      await smartBucket.removeBucket(bucketName);
    } catch {
      // Ignore cleanup errors
    }
  }
});

export default tap.start();