import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as opendata from '../ts/index.js'; /** * Mock provider for testing incremental cache behavior * Allows precise control over what data is returned to test cache logic */ class MockIntradayProvider implements opendata.IStockProvider { name = 'MockIntraday'; priority = 100; requiresAuth = false; // Track fetch calls for testing public fetchCallCount = 0; public lastRequest: opendata.IStockDataRequest | null = null; // Mock data to return private mockData: opendata.IStockPrice[] = []; /** * Set the mock data that will be returned on next fetch */ public setMockData(data: opendata.IStockPrice[]): void { this.mockData = data; } /** * Reset fetch tracking */ public resetTracking(): void { this.fetchCallCount = 0; this.lastRequest = null; } async fetchData(request: opendata.IStockDataRequest): Promise { this.fetchCallCount++; this.lastRequest = request; // For intraday requests, return filtered data based on date if (request.type === 'intraday') { let filteredData = [...this.mockData]; // Filter by date if specified (simulate incremental fetch) if (request.date) { filteredData = filteredData.filter(p => p.timestamp > request.date!); } // Apply limit if (request.limit) { filteredData = filteredData.slice(-request.limit); } return filteredData; } // For other requests, return first item or empty array if (this.mockData.length > 0) { return this.mockData[0]; } throw new Error('No mock data available'); } async isAvailable(): Promise { return true; } } /** * Helper to generate mock intraday prices */ function generateMockIntradayPrices( ticker: string, count: number, startTime: Date, intervalMinutes: number = 1 ): opendata.IStockPrice[] { const prices: opendata.IStockPrice[] = []; let basePrice = 100; for (let i = 0; i < count; i++) { const timestamp = new Date(startTime.getTime() + i * intervalMinutes * 60 * 1000); basePrice += (Math.random() - 0.5) * 2; // Random walk prices.push({ ticker, price: basePrice, currency: 'USD', timestamp, fetchedAt: new Date(), provider: 'MockIntraday', dataType: 'intraday', marketState: 'REGULAR', open: basePrice - 0.5, high: basePrice + 1, low: basePrice - 1, volume: 1000000, change: 0, changePercent: 0, previousClose: basePrice }); } return prices; } let stockService: opendata.StockPriceService; let mockProvider: MockIntradayProvider; tap.test('Incremental Cache Setup', async () => { await tap.test('should create StockPriceService and MockProvider', async () => { stockService = new opendata.StockPriceService({ ttl: 60000, // 1 minute default (will be overridden by smart TTL) maxEntries: 1000 }); expect(stockService).toBeInstanceOf(opendata.StockPriceService); mockProvider = new MockIntradayProvider(); stockService.register(mockProvider); const providers = stockService.getEnabledProviders(); expect(providers).toContainEqual(mockProvider); console.log('✓ Test setup complete'); }); }); tap.test('Incremental Cache - Basic Behavior', async () => { await tap.test('should cache intraday data on first fetch', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); const mockData = generateMockIntradayPrices('AAPL', 10, startTime, 1); mockProvider.setMockData(mockData); // First fetch - should hit provider const result1 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 }); expect(result1).toBeArray(); expect((result1 as opendata.IStockPrice[]).length).toEqual(10); expect(mockProvider.fetchCallCount).toEqual(1); console.log('✓ First fetch cached 10 records'); }); await tap.test('should serve from cache on second identical request', async () => { mockProvider.resetTracking(); // Second fetch - should hit cache (no provider call) const result2 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 }); expect(result2).toBeArray(); expect((result2 as opendata.IStockPrice[]).length).toEqual(10); expect(mockProvider.fetchCallCount).toEqual(0); // Should NOT call provider console.log('✓ Second fetch served from cache (0 provider calls)'); }); }); tap.test('Incremental Cache - Incremental Fetch', async () => { await tap.test('should only fetch NEW data on refresh', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // First fetch: 10 records from 9:30-9:39 const mockData1 = generateMockIntradayPrices('MSFT', 10, startTime, 1); mockProvider.setMockData(mockData1); const result1 = await stockService.getData({ type: 'intraday', ticker: 'MSFT', interval: '1min' }); expect((result1 as opendata.IStockPrice[]).length).toEqual(10); expect(mockProvider.fetchCallCount).toEqual(1); const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp; console.log(`✓ First fetch: 10 records, latest timestamp: ${latestTimestamp1.toISOString()}`); // Simulate 5 minutes passing - 5 new records available mockProvider.resetTracking(); const mockData2 = generateMockIntradayPrices('MSFT', 15, startTime, 1); // 15 total (10 old + 5 new) mockProvider.setMockData(mockData2); // Second fetch - should detect cache and only fetch NEW data const result2 = await stockService.getData({ type: 'intraday', ticker: 'MSFT', interval: '1min' }); expect((result2 as opendata.IStockPrice[]).length).toEqual(15); expect(mockProvider.fetchCallCount).toEqual(1); // Should call provider // Verify the request had a date filter (incremental fetch) expect(mockProvider.lastRequest).not.toEqual(null); expect(mockProvider.lastRequest!.type).toEqual('intraday'); expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined); const requestDate = (mockProvider.lastRequest as opendata.IStockIntradayRequest).date; console.log(`✓ Incremental fetch requested data since: ${requestDate!.toISOString()}`); console.log(`✓ Total records after merge: ${(result2 as opendata.IStockPrice[]).length}`); console.log('✓ Only fetched NEW data (incremental fetch working)'); }); await tap.test('should return cached data when no new records available', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); const mockData = generateMockIntradayPrices('GOOGL', 10, startTime, 1); mockProvider.setMockData(mockData); // First fetch const result1 = await stockService.getData({ type: 'intraday', ticker: 'GOOGL', interval: '1min' }); expect((result1 as opendata.IStockPrice[]).length).toEqual(10); // Second fetch - same data (no new records) mockProvider.resetTracking(); mockProvider.setMockData(mockData); // Same data const result2 = await stockService.getData({ type: 'intraday', ticker: 'GOOGL', interval: '1min' }); expect((result2 as opendata.IStockPrice[]).length).toEqual(10); expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch attempted console.log('✓ No new records - returned cached data'); }); }); tap.test('Incremental Cache - Deduplication', async () => { await tap.test('should deduplicate by timestamp in merged data', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // First fetch: 10 records const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1); mockProvider.setMockData(mockData1); const result1 = await stockService.getData({ type: 'intraday', ticker: 'TSLA', interval: '1min' }); expect((result1 as opendata.IStockPrice[]).length).toEqual(10); // Second fetch: Return overlapping data (last 5 old + 5 new) // This simulates provider returning some duplicate timestamps mockProvider.resetTracking(); const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1); mockProvider.setMockData(mockData2); const result2 = await stockService.getData({ type: 'intraday', ticker: 'TSLA', interval: '1min' }); // Should have 15 unique timestamps (deduplication worked) expect((result2 as opendata.IStockPrice[]).length).toEqual(15); // Verify timestamps are unique const timestamps = (result2 as opendata.IStockPrice[]).map(p => p.timestamp.getTime()); const uniqueTimestamps = new Set(timestamps); expect(uniqueTimestamps.size).toEqual(15); console.log('✓ Deduplication working - 15 unique timestamps'); }); }); tap.test('Incremental Cache - Limit Handling', async () => { await tap.test('should respect limit parameter in merged results', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // First fetch with limit 100 const mockData1 = generateMockIntradayPrices('AMZN', 100, startTime, 1); mockProvider.setMockData(mockData1); const result1 = await stockService.getData({ type: 'intraday', ticker: 'AMZN', interval: '1min', limit: 100 }); expect((result1 as opendata.IStockPrice[]).length).toEqual(100); // Second fetch: 10 new records available mockProvider.resetTracking(); const mockData2 = generateMockIntradayPrices('AMZN', 110, startTime, 1); mockProvider.setMockData(mockData2); const result2 = await stockService.getData({ type: 'intraday', ticker: 'AMZN', interval: '1min', limit: 100 // Same limit }); // Should still return 100 (most recent 100 after merge) expect((result2 as opendata.IStockPrice[]).length).toEqual(100); // Verify we got the most RECENT 100 (should include new data) const lastTimestamp = (result2 as opendata.IStockPrice[])[99].timestamp; const expectedLastTimestamp = mockData2[109].timestamp; expect(lastTimestamp.getTime()).toEqual(expectedLastTimestamp.getTime()); console.log('✓ Limit respected - returned most recent 100 records'); }); await tap.test('should handle different limits without cache collision', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); const mockData = generateMockIntradayPrices('NVDA', 1000, startTime, 1); mockProvider.setMockData(mockData); // Fetch with limit 100 const result1 = await stockService.getData({ type: 'intraday', ticker: 'NVDA', interval: '1min', limit: 100 }); expect((result1 as opendata.IStockPrice[]).length).toEqual(100); mockProvider.resetTracking(); // Fetch with limit 500 (should NOT use cached limit:100 data) const result2 = await stockService.getData({ type: 'intraday', ticker: 'NVDA', interval: '1min', limit: 500 }); expect((result2 as opendata.IStockPrice[]).length).toEqual(500); // Should have made a new provider call (different cache key) expect(mockProvider.fetchCallCount).toBeGreaterThan(0); console.log('✓ Different limits use different cache keys'); }); }); tap.test('Incremental Cache - Dashboard Polling Scenario', async () => { await tap.test('should efficiently handle repeated polling requests', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); let currentDataSize = 100; // Initial fetch: 100 records let mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1); mockProvider.setMockData(mockData); const result1 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 1000 }); expect((result1 as opendata.IStockPrice[]).length).toEqual(100); const initialFetchCount = mockProvider.fetchCallCount; console.log(`✓ Initial fetch: ${(result1 as opendata.IStockPrice[]).length} records (${initialFetchCount} API calls)`); // Simulate 5 dashboard refreshes (1 new record each time) let totalNewRecords = 0; for (let i = 0; i < 5; i++) { mockProvider.resetTracking(); currentDataSize += 1; // 1 new record totalNewRecords += 1; mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1); mockProvider.setMockData(mockData); const result = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 1000 }); expect((result as opendata.IStockPrice[]).length).toEqual(currentDataSize); expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch } console.log(`✓ Dashboard polling: 5 refreshes with ${totalNewRecords} new records`); console.log('✓ Each refresh only fetched NEW data (incremental cache working)'); }); }); tap.test('Incremental Cache - Memory Impact', async () => { await tap.test('should demonstrate memory savings from deduplication', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // Create data with intentional duplicates const baseData = generateMockIntradayPrices('MSFT', 1000, startTime, 1); const duplicatedData = [...baseData, ...baseData.slice(-100)]; // Duplicate last 100 expect(duplicatedData.length).toEqual(1100); // Before deduplication mockProvider.setMockData(duplicatedData); const result = await stockService.getData({ type: 'intraday', ticker: 'MSFT', interval: '1min' }); // Should have 1000 unique records (100 duplicates removed) expect((result as opendata.IStockPrice[]).length).toEqual(1000); console.log('✓ Deduplication removed 100 duplicate timestamps'); console.log(`✓ Memory saved: ~${Math.round((100 / 1100) * 100)}%`); }); }); tap.test('Incremental Cache - Fallback Behavior', async () => { await tap.test('should not use incremental fetch for requests with date filter', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); const mockData = generateMockIntradayPrices('GOOGL', 100, startTime, 1); mockProvider.setMockData(mockData); // First fetch without date await stockService.getData({ type: 'intraday', ticker: 'GOOGL', interval: '1min' }); mockProvider.resetTracking(); // Second fetch WITH date filter - should NOT use incremental cache const result = await stockService.getData({ type: 'intraday', ticker: 'GOOGL', interval: '1min', date: new Date('2025-01-07T10:00:00.000Z') // Explicit date filter }); // Should have made normal fetch (not incremental) expect(mockProvider.fetchCallCount).toEqual(1); expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined); console.log('✓ Incremental cache skipped for requests with explicit date filter'); }); }); tap.test('Incremental Cache - Performance Benchmark', async () => { await tap.test('should demonstrate API call reduction', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // Initial dataset: 1000 records let mockData = generateMockIntradayPrices('BENCHMARK', 1000, startTime, 1); mockProvider.setMockData(mockData); // Initial fetch await stockService.getData({ type: 'intraday', ticker: 'BENCHMARK', interval: '1min', limit: 1000 }); expect(mockProvider.fetchCallCount).toEqual(1); console.log('✓ Initial fetch: 1000 records'); let totalProviderCalls = 1; let totalNewRecords = 0; // Simulate 10 refreshes (5 new records each) for (let i = 0; i < 10; i++) { mockProvider.resetTracking(); // Add 5 new records const newCount = 5; mockData = generateMockIntradayPrices('BENCHMARK', 1000 + totalNewRecords + newCount, startTime, 1); mockProvider.setMockData(mockData); await stockService.getData({ type: 'intraday', ticker: 'BENCHMARK', interval: '1min', limit: 1000 }); totalProviderCalls += mockProvider.fetchCallCount; totalNewRecords += newCount; } console.log('\n📊 Performance Benchmark:'); console.log(` Total refreshes: 10`); console.log(` New records fetched: ${totalNewRecords}`); console.log(` Total provider calls: ${totalProviderCalls}`); console.log(` Without incremental cache: ${11} calls (1 initial + 10 full refreshes)`); console.log(` With incremental cache: ${totalProviderCalls} calls (1 initial + 10 incremental)`); console.log(` Data transfer reduction: ~${Math.round((1 - (totalNewRecords / (10 * 1000))) * 100)}%`); console.log(' (Only fetched NEW data instead of refetching all 1000 records each time)'); }); }); tap.test('Incremental Cache - Timestamp Ordering', async () => { await tap.test('should maintain timestamp order after merge', async () => { stockService.clearCache(); mockProvider.resetTracking(); const startTime = new Date('2025-01-07T09:30:00.000Z'); // First fetch const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1); mockProvider.setMockData(mockData1); await stockService.getData({ type: 'intraday', ticker: 'TSLA', interval: '1min' }); // Second fetch with new data mockProvider.resetTracking(); const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1); mockProvider.setMockData(mockData2); const result = await stockService.getData({ type: 'intraday', ticker: 'TSLA', interval: '1min' }); // Verify ascending timestamp order const timestamps = (result as opendata.IStockPrice[]).map(p => p.timestamp.getTime()); for (let i = 1; i < timestamps.length; i++) { expect(timestamps[i]).toBeGreaterThan(timestamps[i - 1]); } console.log('✓ Timestamps correctly ordered (ascending)'); }); }); export default tap.start();