opendata/test/test.stale-data-fix.node+bun+deno.ts

import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';

/**
 * Test to verify we NEVER return stale intraday data
 * Even when cache hasn't expired, we should check for new data
 */

class MockIntradayProvider implements opendata.IStockProvider {
  name = 'MockIntradayProvider';
  priority = 100;
  requiresAuth = false;

  public fetchCount = 0;
  public lastRequestDate: Date | undefined;
  private currentDataCount = 10; // Start with 10 records
  private baseTime = new Date('2025-01-07T09:30:00.000Z');

  async fetchData(request: opendata.IStockDataRequest): Promise<opendata.IStockPrice | opendata.IStockPrice[]> {
    this.fetchCount++;

    if (request.type === 'intraday') {
      this.lastRequestDate = request.date;

      const startTime = request.date || this.baseTime;
      const prices: opendata.IStockPrice[] = [];

      // Simulate provider returning data AFTER the requested date
      for (let i = 0; i < this.currentDataCount; i++) {
        const timestamp = new Date(startTime.getTime() + i * 60 * 1000);

        // Only return data AFTER request date if date filter is present
        if (request.date && timestamp <= request.date) {
          continue;
        }

        prices.push({
          ticker: request.ticker,
          price: 100 + i,
          currency: 'USD',
          timestamp,
          fetchedAt: new Date(),
          provider: this.name,
          dataType: 'intraday',
          marketState: 'REGULAR',
          open: 100,
          high: 101,
          low: 99,
          volume: 1000000,
          change: 0,
          changePercent: 0,
          previousClose: 100
        });
      }

      return prices;
    }

    throw new Error('Only intraday supported in this mock');
  }

  async isAvailable(): Promise<boolean> {
    return true;
  }

  public addNewRecords(count: number): void {
    this.currentDataCount += count;
  }

  public advanceTime(minutes: number): void {
    this.baseTime = new Date(this.baseTime.getTime() + minutes * 60 * 1000);
  }
}

let stockService: opendata.StockPriceService;
let mockProvider: MockIntradayProvider;

tap.test('Stale Data Fix - Setup', async () => {
  // Use LONG TTL so cache doesn't expire during test
  stockService = new opendata.StockPriceService({
    ttl: 300000, // 5 minutes
    maxEntries: 1000
  });

  mockProvider = new MockIntradayProvider();
  stockService.register(mockProvider);

  console.log('✓ Service initialized with 5-minute cache TTL');
});

tap.test('Stale Data Fix - Check for New Data Even When Cache Valid', async () => {
  await tap.test('should return cached data if less than 1 minute old (freshness check)', async () => {
    stockService.clearCache();
    mockProvider.fetchCount = 0;
    mockProvider.currentDataCount = 10;

    console.log('\n📊 Scenario: Request twice within 1 minute\n');

    // First request - fetch 10 records
    console.log('⏰ First request (initial fetch)');
    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'AAPL',
      interval: '1min',
      limit: 1000
    });

    expect(result1).toBeArray();
    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCount).toEqual(1);

    const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
    console.log(`  ✓ Fetched 10 records, latest: ${latestTimestamp1.toISOString()}`);

    // Second request immediately - should return cache (data < 1min old)
    console.log('\n⏰ Second request (< 1 minute later)');
    mockProvider.fetchCount = 0;
    mockProvider.addNewRecords(10); // New data available, but won't fetch yet

    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'AAPL',
      interval: '1min',
      limit: 1000
    });

    // Should return cached data (freshness check prevents fetch)
    expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
    expect(mockProvider.fetchCount).toEqual(0); // No provider call

    console.log(`  ✓ Returned cached 10 records (no provider call)`);
    console.log(`  ✓ Freshness check: Data < 1min old, no fetch needed`);
  });

  await tap.test('should fetch NEW data when cache is > 1 minute old', async () => {
    stockService.clearCache();
    mockProvider.fetchCount = 0;
    mockProvider.currentDataCount = 10;

    console.log('\n📊 Scenario: Request after 2 minutes (data > 1min old)\n');

    // First request - fetch 10 records at 9:30am
    console.log('⏰ 9:30:00 - First request (initial fetch)');
    const result1 = await stockService.getData({
      type: 'intraday',
      ticker: 'MSFT',
      interval: '1min',
      limit: 1000
    });

    expect(result1).toBeArray();
    expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
    const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
    console.log(`  ✓ Fetched 10 records, latest: ${latestTimestamp1.toISOString()}`);

    // Advance time by 2 minutes - now data is > 1 minute old
    console.log('\n⏰ 9:32:00 - Second request (2 minutes later, data > 1min old)');
    console.log('  📝 Advancing provider time by 2 minutes...');

    mockProvider.fetchCount = 0;
    mockProvider.advanceTime(2); // Advance 2 minutes
    mockProvider.addNewRecords(10); // Now provider has 20 records total

    const result2 = await stockService.getData({
      type: 'intraday',
      ticker: 'MSFT',
      interval: '1min',
      limit: 1000
    });

    expect(result2).toBeArray();
    const prices2 = result2 as opendata.IStockPrice[];

    // Should have 20 records (10 cached + 10 new)
    expect(prices2.length).toEqual(20);

    // Should have made a provider call (data was stale)
    expect(mockProvider.fetchCount).toBeGreaterThan(0);

    const latestTimestamp2 = prices2[prices2.length - 1].timestamp;
    console.log(`  ✓ Now have ${prices2.length} records, latest: ${latestTimestamp2.toISOString()}`);
    console.log(`  ✓ Provider calls: ${mockProvider.fetchCount} (fetched new data)`);
    console.log(`  ✓ Data was > 1min old, incremental fetch triggered!`);

    // Verify we got NEW data
    expect(latestTimestamp2.getTime()).toBeGreaterThan(latestTimestamp1.getTime());

    console.log('\n✅ SUCCESS: Fetched new data when cache was stale!');
  });

  await tap.test('should handle polling with > 1 minute intervals efficiently', async () => {
    stockService.clearCache();
    mockProvider.fetchCount = 0;
    mockProvider.currentDataCount = 100;

    console.log('\n📊 Scenario: Dashboard polling every 2 minutes\n');

    // Initial request at 9:30am
    console.log('⏰ 9:30:00 - Request 1 (initial fetch)');
    await stockService.getData({
      type: 'intraday',
      ticker: 'GOOGL',
      interval: '1min',
      limit: 1000
    });

    expect(mockProvider.fetchCount).toEqual(1);
    console.log(`  ✓ Fetched 100 records (provider calls: 1)`);

    let totalProviderCalls = 1;
    let totalNewRecords = 0;

    // Simulate 3 polling refreshes (2 minutes apart, 5 new records each)
    for (let i = 2; i <= 4; i++) {
      mockProvider.fetchCount = 0;
      mockProvider.advanceTime(2); // Advance 2 minutes (triggers freshness check)
      mockProvider.addNewRecords(5);
      totalNewRecords += 5;

      const minutes = (i - 1) * 2;
      console.log(`\n⏰ 9:${30 + minutes}:00 - Request ${i} (${minutes} minutes later, +5 new records)`);

      const result = await stockService.getData({
        type: 'intraday',
        ticker: 'GOOGL',
        interval: '1min',
        limit: 1000
      });

      const expectedTotal = 100 + totalNewRecords;
      expect((result as opendata.IStockPrice[]).length).toEqual(expectedTotal);

      // Should have made exactly 1 provider call (incremental fetch)
      expect(mockProvider.fetchCount).toEqual(1);
      totalProviderCalls++;

      console.log(`  ✓ Now have ${expectedTotal} records (incremental fetch: 1 call)`);
    }

    console.log(`\n📊 Summary:`);
    console.log(`  Total requests: 4`);
    console.log(`  Total provider calls: ${totalProviderCalls}`);
    console.log(`  New records fetched: ${totalNewRecords}`);
    console.log(`  Without incremental cache: Would fetch 100 records × 3 refreshes = 300 records`);
    console.log(`  With incremental cache: Only fetched ${totalNewRecords} new records`);
    console.log(`  Data transfer reduction: ${Math.round((1 - (totalNewRecords / 300)) * 100)}%`);
    console.log('\n✅ SUCCESS: Only fetched NEW data on each refresh!');
  });
});

tap.test('Stale Data Fix - Verify No Regression for Other Request Types', async () => {
  await tap.test('historical requests should still use simple cache', async () => {
    stockService.clearCache();

    // Mock provider that counts calls
    let historicalCallCount = 0;
    const historicalProvider: opendata.IStockProvider = {
      name: 'HistoricalMock',
      priority: 100,
      requiresAuth: false,
      async fetchData() {
        historicalCallCount++;
        return [{
          ticker: 'TEST',
          price: 100,
          currency: 'USD',
          timestamp: new Date('2025-01-01'),
          fetchedAt: new Date(),
          provider: 'HistoricalMock',
          dataType: 'eod',
          marketState: 'CLOSED',
          open: 99,
          high: 101,
          low: 98,
          volume: 1000000,
          change: 1,
          changePercent: 1,
          previousClose: 99
        }];
      },
      async isAvailable() { return true; }
    };

    const testService = new opendata.StockPriceService({ ttl: 60000 });
    testService.register(historicalProvider);

    // First request
    await testService.getData({
      type: 'historical',
      ticker: 'TEST',
      from: new Date('2025-01-01'),
      to: new Date('2025-01-31')
    });

    expect(historicalCallCount).toEqual(1);

    // Second request - should use cache (not incremental fetch)
    await testService.getData({
      type: 'historical',
      ticker: 'TEST',
      from: new Date('2025-01-01'),
      to: new Date('2025-01-31')
    });

    // Should still be 1 (used cache)
    expect(historicalCallCount).toEqual(1);

    console.log('✓ Historical requests use simple cache (no incremental fetch)');
  });

  await tap.test('current price requests should still use simple cache', async () => {
    stockService.clearCache();

    let currentCallCount = 0;
    const currentProvider: opendata.IStockProvider = {
      name: 'CurrentMock',
      priority: 100,
      requiresAuth: false,
      async fetchData() {
        currentCallCount++;
        return {
          ticker: 'TEST',
          price: 150,
          currency: 'USD',
          timestamp: new Date(),
          fetchedAt: new Date(),
          provider: 'CurrentMock',
          dataType: 'eod',
          marketState: 'CLOSED',
          open: 149,
          high: 151,
          low: 148,
          volume: 5000000,
          change: 1,
          changePercent: 0.67,
          previousClose: 149
        };
      },
      async isAvailable() { return true; }
    };

    const testService = new opendata.StockPriceService({ ttl: 60000 });
    testService.register(currentProvider);

    // First request
    await testService.getData({
      type: 'current',
      ticker: 'TEST'
    });

    expect(currentCallCount).toEqual(1);

    // Second request - should use cache
    await testService.getData({
      type: 'current',
      ticker: 'TEST'
    });

    expect(currentCallCount).toEqual(1);

    console.log('✓ Current price requests use simple cache');
  });
});

export default tap.start();