opendata/test/test.incremental-cache.node+bun+deno.ts

import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';

/**
 * Mock provider for testing incremental cache behavior
 * Allows precise control over what data is returned to test cache logic
 */
class MockIntradayProvider implements opendata.IStockProvider {
  name = 'MockIntraday';
  priority = 100;
  requiresAuth = false;

  // Track fetch calls for testing
  public fetchCallCount = 0;
  public lastRequest: opendata.IStockDataRequest | null = null;

  // Mock data to return
  private mockData: opendata.IStockPrice[] = [];

  /**
   * Set the mock data that will be returned on next fetch
   */
  public setMockData(data: opendata.IStockPrice[]): void {
    this.mockData = data;
  }

  /**
   * Reset fetch tracking
   */
  public resetTracking(): void {
    this.fetchCallCount = 0;
    this.lastRequest = null;
  }

  async fetchData(request: opendata.IStockDataRequest): Promise<opendata.IStockPrice | opendata.IStockPrice[]> {
    this.fetchCallCount++;
    this.lastRequest = request;

    // For intraday requests, return filtered data based on date
    if (request.type === 'intraday') {
      let filteredData = [...this.mockData];

      // Filter by date if specified (simulate incremental fetch)
      if (request.date) {
        filteredData = filteredData.filter(p => p.timestamp > request.date!);
      }

      // Apply limit
      if (request.limit) {
        filteredData = filteredData.slice(-request.limit);
      }

      return filteredData;
    }

    // For other requests, return first item or empty array
    if (this.mockData.length > 0) {
      return this.mockData[0];
    }

    throw new Error('No mock data available');
  }

  async isAvailable(): Promise<boolean> {
    return true;
  }
}

/**
 * Helper to generate mock intraday prices
 */
function generateMockIntradayPrices(
  ticker: string,
  count: number,
  startTime: Date,
  intervalMinutes: number = 1
): opendata.IStockPrice[] {
  const prices: opendata.IStockPrice[] = [];
  let basePrice = 100;

  for (let i = 0; i < count; i++) {
    const timestamp = new Date(startTime.getTime() + i * intervalMinutes * 60 * 1000);
    basePrice += (Math.random() - 0.5) * 2; // Random walk

    prices.push({
      ticker,
      price: basePrice,
      currency: 'USD',
      timestamp,
      fetchedAt: new Date(),
      provider: 'MockIntraday',
      dataType: 'intraday',
      marketState: 'REGULAR',
      open: basePrice - 0.5,
      high: basePrice + 1,
      low: basePrice - 1,
      volume: 1000000,
      change: 0,
      changePercent: 0,
      previousClose: basePrice
    });
  }

  return prices;
}

let stockService: opendata.StockPriceService;
let mockProvider: MockIntradayProvider;

tap.test('Incremental Cache Setup', async () => {
  await tap.test('should create StockPriceService and MockProvider', async () => {
    stockService = new opendata.StockPriceService({
      ttl: 60000, // 1 minute default (will be overridden by smart TTL)
      maxEntries: 1000
    });
    expect(stockService).toBeInstanceOf(opendata.StockPriceService);

    mockProvider = new MockIntradayProvider();
    stockService.register(mockProvider);

    const providers = stockService.getEnabledProviders();
    expect(providers).toContainEqual(mockProvider);

    console.log('✓ Test setup complete');
  });
});

tap.test('Incremental Cache - Basic Behavior', async () => {
  await tap.test('should cache intraday data on first fetch', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');
    const mockData = generateMockIntradayPrices('AAPL', 10, startTime, 1);
    mockProvider.setMockData(mockData);

    // First fetch - should hit provider
    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'AAPL',
      interval: '1min',
      limit: 10
    });

    expect(result1).toBeArray();
    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCallCount).toEqual(1);

    console.log('✓ First fetch cached 10 records');
  });

  await tap.test('should serve from cache on second identical request', async () => {
    mockProvider.resetTracking();

    // Second fetch - should hit cache (no provider call)
    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'AAPL',
      interval: '1min',
      limit: 10
    });

    expect(result2).toBeArray();
    expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCallCount).toEqual(0); // Should NOT call provider

    console.log('✓ Second fetch served from cache (0 provider calls)');
  });
});

tap.test('Incremental Cache - Incremental Fetch', async () => {
  await tap.test('should only fetch NEW data on refresh', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // First fetch: 10 records from 9:30-9:39
    const mockData1 = generateMockIntradayPrices('MSFT', 10, startTime, 1);
    mockProvider.setMockData(mockData1);

    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'MSFT',
      interval: '1min'
    });

    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCallCount).toEqual(1);

    const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
    console.log(`✓ First fetch: 10 records, latest timestamp: ${latestTimestamp1.toISOString()}`);

    // Simulate 5 minutes passing - 5 new records available
    mockProvider.resetTracking();
    const mockData2 = generateMockIntradayPrices('MSFT', 15, startTime, 1); // 15 total (10 old + 5 new)
    mockProvider.setMockData(mockData2);

    // Second fetch - should detect cache and only fetch NEW data
    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'MSFT',
      interval: '1min'
    });

    expect((result2 as opendata.IStockPrice[]).length).toEqual(15);
    expect(mockProvider.fetchCallCount).toEqual(1); // Should call provider

    // Verify the request had a date filter (incremental fetch)
    expect(mockProvider.lastRequest).not.toEqual(null);
    expect(mockProvider.lastRequest!.type).toEqual('intraday');
    expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined);

    const requestDate = (mockProvider.lastRequest as opendata.IStockIntradayRequest).date;
    console.log(`✓ Incremental fetch requested data since: ${requestDate!.toISOString()}`);
    console.log(`✓ Total records after merge: ${(result2 as opendata.IStockPrice[]).length}`);
    console.log('✓ Only fetched NEW data (incremental fetch working)');
  });

  await tap.test('should return cached data when no new records available', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');
    const mockData = generateMockIntradayPrices('GOOGL', 10, startTime, 1);
    mockProvider.setMockData(mockData);

    // First fetch
    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'GOOGL',
      interval: '1min'
    });

    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);

    // Second fetch - same data (no new records)
    mockProvider.resetTracking();
    mockProvider.setMockData(mockData); // Same data

    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'GOOGL',
      interval: '1min'
    });

    expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch attempted

    console.log('✓ No new records - returned cached data');
  });
});

tap.test('Incremental Cache - Deduplication', async () => {
  await tap.test('should deduplicate by timestamp in merged data', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // First fetch: 10 records
    const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1);
    mockProvider.setMockData(mockData1);

    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'TSLA',
      interval: '1min'
    });

    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);

    // Second fetch: Return overlapping data (last 5 old + 5 new)
    // This simulates provider returning some duplicate timestamps
    mockProvider.resetTracking();
    const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1);
    mockProvider.setMockData(mockData2);

    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'TSLA',
      interval: '1min'
    });

    // Should have 15 unique timestamps (deduplication worked)
    expect((result2 as opendata.IStockPrice[]).length).toEqual(15);

    // Verify timestamps are unique
    const timestamps = (result2 as opendata.IStockPrice[]).map(p => p.timestamp.getTime());
    const uniqueTimestamps = new Set(timestamps);
    expect(uniqueTimestamps.size).toEqual(15);

    console.log('✓ Deduplication working - 15 unique timestamps');
  });
});

tap.test('Incremental Cache - Limit Handling', async () => {
  await tap.test('should respect limit parameter in merged results', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // First fetch with limit 100
    const mockData1 = generateMockIntradayPrices('AMZN', 100, startTime, 1);
    mockProvider.setMockData(mockData1);

    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'AMZN',
      interval: '1min',
      limit: 100
    });

    expect((result1 as opendata.IStockPrice[]).length).toEqual(100);

    // Second fetch: 10 new records available
    mockProvider.resetTracking();
    const mockData2 = generateMockIntradayPrices('AMZN', 110, startTime, 1);
    mockProvider.setMockData(mockData2);

    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'AMZN',
      interval: '1min',
      limit: 100 // Same limit
    });

    // Should still return 100 (most recent 100 after merge)
    expect((result2 as opendata.IStockPrice[]).length).toEqual(100);

    // Verify we got the most RECENT 100 (should include new data)
    const lastTimestamp = (result2 as opendata.IStockPrice[])[99].timestamp;
    const expectedLastTimestamp = mockData2[109].timestamp;
    expect(lastTimestamp.getTime()).toEqual(expectedLastTimestamp.getTime());

    console.log('✓ Limit respected - returned most recent 100 records');
  });

  await tap.test('should handle different limits without cache collision', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');
    const mockData = generateMockIntradayPrices('NVDA', 1000, startTime, 1);
    mockProvider.setMockData(mockData);

    // Fetch with limit 100
    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'NVDA',
      interval: '1min',
      limit: 100
    });
    expect((result1 as opendata.IStockPrice[]).length).toEqual(100);

    mockProvider.resetTracking();

    // Fetch with limit 500 (should NOT use cached limit:100 data)
    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'NVDA',
      interval: '1min',
      limit: 500
    });
    expect((result2 as opendata.IStockPrice[]).length).toEqual(500);

    // Should have made a new provider call (different cache key)
    expect(mockProvider.fetchCallCount).toBeGreaterThan(0);

    console.log('✓ Different limits use different cache keys');
  });
});

tap.test('Incremental Cache - Dashboard Polling Scenario', async () => {
  await tap.test('should efficiently handle repeated polling requests', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');
    let currentDataSize = 100;

    // Initial fetch: 100 records
    let mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1);
    mockProvider.setMockData(mockData);

    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'AAPL',
      interval: '1min',
      limit: 1000
    });

    expect((result1 as opendata.IStockPrice[]).length).toEqual(100);
    const initialFetchCount = mockProvider.fetchCallCount;

    console.log(`✓ Initial fetch: ${(result1 as opendata.IStockPrice[]).length} records (${initialFetchCount} API calls)`);

    // Simulate 5 dashboard refreshes (1 new record each time)
    let totalNewRecords = 0;
    for (let i = 0; i < 5; i++) {
      mockProvider.resetTracking();
      currentDataSize += 1; // 1 new record
      totalNewRecords += 1;

      mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1);
      mockProvider.setMockData(mockData);

      const result = await stockService.getData({
        type: 'intraday',
        ticker: 'AAPL',
        interval: '1min',
        limit: 1000
      });

      expect((result as opendata.IStockPrice[]).length).toEqual(currentDataSize);
      expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch
    }

    console.log(`✓ Dashboard polling: 5 refreshes with ${totalNewRecords} new records`);
    console.log('✓ Each refresh only fetched NEW data (incremental cache working)');
  });
});

tap.test('Incremental Cache - Memory Impact', async () => {
  await tap.test('should demonstrate memory savings from deduplication', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // Create data with intentional duplicates
    const baseData = generateMockIntradayPrices('MSFT', 1000, startTime, 1);
    const duplicatedData = [...baseData, ...baseData.slice(-100)]; // Duplicate last 100

    expect(duplicatedData.length).toEqual(1100); // Before deduplication

    mockProvider.setMockData(duplicatedData);

    const result = await stockService.getData({
      type: 'intraday',
      ticker: 'MSFT',
      interval: '1min'
    });

    // Should have 1000 unique records (100 duplicates removed)
    expect((result as opendata.IStockPrice[]).length).toEqual(1000);

    console.log('✓ Deduplication removed 100 duplicate timestamps');
    console.log(`✓ Memory saved: ~${Math.round((100 / 1100) * 100)}%`);
  });
});

tap.test('Incremental Cache - Fallback Behavior', async () => {
  await tap.test('should not use incremental fetch for requests with date filter', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');
    const mockData = generateMockIntradayPrices('GOOGL', 100, startTime, 1);
    mockProvider.setMockData(mockData);

    // First fetch without date
    await stockService.getData({
      type: 'intraday',
      ticker: 'GOOGL',
      interval: '1min'
    });

    mockProvider.resetTracking();

    // Second fetch WITH date filter - should NOT use incremental cache
    const result = await stockService.getData({
      type: 'intraday',
      ticker: 'GOOGL',
      interval: '1min',
      date: new Date('2025-01-07T10:00:00.000Z') // Explicit date filter
    });

    // Should have made normal fetch (not incremental)
    expect(mockProvider.fetchCallCount).toEqual(1);
    expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined);

    console.log('✓ Incremental cache skipped for requests with explicit date filter');
  });
});

tap.test('Incremental Cache - Performance Benchmark', async () => {
  await tap.test('should demonstrate API call reduction', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // Initial dataset: 1000 records
    let mockData = generateMockIntradayPrices('BENCHMARK', 1000, startTime, 1);
    mockProvider.setMockData(mockData);

    // Initial fetch
    await stockService.getData({
      type: 'intraday',
      ticker: 'BENCHMARK',
      interval: '1min',
      limit: 1000
    });

    expect(mockProvider.fetchCallCount).toEqual(1);
    console.log('✓ Initial fetch: 1000 records');

    let totalProviderCalls = 1;
    let totalNewRecords = 0;

    // Simulate 10 refreshes (5 new records each)
    for (let i = 0; i < 10; i++) {
      mockProvider.resetTracking();

      // Add 5 new records
      const newCount = 5;
      mockData = generateMockIntradayPrices('BENCHMARK', 1000 + totalNewRecords + newCount, startTime, 1);
      mockProvider.setMockData(mockData);

      await stockService.getData({
        type: 'intraday',
        ticker: 'BENCHMARK',
        interval: '1min',
        limit: 1000
      });

      totalProviderCalls += mockProvider.fetchCallCount;
      totalNewRecords += newCount;
    }

    console.log('\n📊 Performance Benchmark:');
    console.log(`  Total refreshes: 10`);
    console.log(`  New records fetched: ${totalNewRecords}`);
    console.log(`  Total provider calls: ${totalProviderCalls}`);
    console.log(`  Without incremental cache: ${11} calls (1 initial + 10 full refreshes)`);
    console.log(`  With incremental cache: ${totalProviderCalls} calls (1 initial + 10 incremental)`);
    console.log(`  Data transfer reduction: ~${Math.round((1 - (totalNewRecords / (10 * 1000))) * 100)}%`);
    console.log('  (Only fetched NEW data instead of refetching all 1000 records each time)');
  });
});

tap.test('Incremental Cache - Timestamp Ordering', async () => {
  await tap.test('should maintain timestamp order after merge', async () => {
    stockService.clearCache();
    mockProvider.resetTracking();

    const startTime = new Date('2025-01-07T09:30:00.000Z');

    // First fetch
    const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1);
    mockProvider.setMockData(mockData1);

    await stockService.getData({
      type: 'intraday',
      ticker: 'TSLA',
      interval: '1min'
    });

    // Second fetch with new data
    mockProvider.resetTracking();
    const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1);
    mockProvider.setMockData(mockData2);

    const result = await stockService.getData({
      type: 'intraday',
      ticker: 'TSLA',
      interval: '1min'
    });

    // Verify ascending timestamp order
    const timestamps = (result as opendata.IStockPrice[]).map(p => p.timestamp.getTime());
    for (let i = 1; i < timestamps.length; i++) {
      expect(timestamps[i]).toBeGreaterThan(timestamps[i - 1]);
    }

    console.log('✓ Timestamps correctly ordered (ascending)');
  });
});

export default tap.start();