opendata/ts/stocks/classes.stockservice.ts

import * as plugins from '../plugins.js';
import type { IStockProvider, IProviderConfig, IProviderRegistry } from './interfaces/provider.js';
import type {
  IStockPrice,
  IStockPriceError,
  IStockDataRequest,
  IStockCurrentRequest,
  IStockHistoricalRequest,
  IStockIntradayRequest,
  IStockBatchCurrentRequest,
  TIntervalType
} from './interfaces/stockprice.js';

// Simple request interfaces for convenience methods
interface ISimpleQuoteRequest {
  ticker: string;
}

interface ISimpleBatchRequest {
  tickers: string[];
}

interface IProviderEntry {
  provider: IStockProvider;
  config: IProviderConfig;
  lastError?: Error;
  lastErrorTime?: Date;
  successCount: number;
  errorCount: number;
}

interface ICacheEntry {
  price: IStockPrice | IStockPrice[];
  timestamp: Date;
  ttl: number; // Specific TTL for this entry
}

export class StockPriceService implements IProviderRegistry {
  private providers = new Map<string, IProviderEntry>();
  private cache = new Map<string, ICacheEntry>();
  private logger = console;

  private cacheConfig = {
    ttl: 60000, // 60 seconds default (for backward compatibility)
    maxEntries: 10000 // Increased for historical data
  };

  constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
    if (cacheConfig) {
      this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
    }
  }

  /**
   * Get data-type aware TTL for smart caching
   */
  private getCacheTTL(dataType: 'eod' | 'historical' | 'intraday' | 'live', interval?: TIntervalType): number {
    switch (dataType) {
      case 'historical':
        return Infinity; // Historical data never changes
      case 'eod':
        return 24 * 60 * 60 * 1000; // 24 hours (EOD is static after market close)
      case 'intraday':
        // Match cache TTL to interval
        return this.getIntervalMs(interval);
      case 'live':
        return 30 * 1000; // 30 seconds for live data
      default:
        return this.cacheConfig.ttl; // Fallback to default
    }
  }

  /**
   * Convert interval to milliseconds
   */
  private getIntervalMs(interval?: TIntervalType): number {
    if (!interval) return 60 * 1000; // Default 1 minute

    const intervalMap: Record<TIntervalType, number> = {
      '1min': 60 * 1000,
      '5min': 5 * 60 * 1000,
      '10min': 10 * 60 * 1000,
      '15min': 15 * 60 * 1000,
      '30min': 30 * 60 * 1000,
      '1hour': 60 * 60 * 1000
    };

    return intervalMap[interval] || 60 * 1000;
  }

  public register(provider: IStockProvider, config?: IProviderConfig): void {
    const defaultConfig: IProviderConfig = {
      enabled: true,
      priority: provider.priority,
      timeout: 10000,
      retryAttempts: 2,
      retryDelay: 1000
    };

    const mergedConfig = { ...defaultConfig, ...config };
    
    this.providers.set(provider.name, {
      provider,
      config: mergedConfig,
      successCount: 0,
      errorCount: 0
    });

    console.log(`Registered provider: ${provider.name}`);
  }

  public unregister(providerName: string): void {
    this.providers.delete(providerName);
    console.log(`Unregistered provider: ${providerName}`);
  }

  public getProvider(name: string): IStockProvider | undefined {
    return this.providers.get(name)?.provider;
  }

  public getAllProviders(): IStockProvider[] {
    return Array.from(this.providers.values()).map(entry => entry.provider);
  }

  public getEnabledProviders(): IStockProvider[] {
    return Array.from(this.providers.values())
      .filter(entry => entry.config.enabled)
      .sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
      .map(entry => entry.provider);
  }

  /**
   * Convenience method: Get current price for a single ticker
   */
  public async getPrice(request: ISimpleQuoteRequest): Promise<IStockPrice> {
    const result = await this.getData({
      type: 'current',
      ticker: request.ticker
    });
    return result as IStockPrice;
  }

  /**
   * Convenience method: Get current prices for multiple tickers
   */
  public async getPrices(request: ISimpleBatchRequest): Promise<IStockPrice[]> {
    const result = await this.getData({
      type: 'batch',
      tickers: request.tickers
    });
    return result as IStockPrice[];
  }

  /**
   * New unified data fetching method supporting all request types
   */
  public async getData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
    const cacheKey = this.getDataCacheKey(request);

    // For intraday requests without date filter, ALWAYS try incremental fetch
    // This ensures we check for new data even if cache hasn't expired
    if (request.type === 'intraday' && !request.date) {
      const incrementalResult = await this.tryIncrementalFetch(request, cacheKey);
      if (incrementalResult) {
        return incrementalResult;
      }
      // If incremental fetch returns null, continue to normal fetch below
    } else {
      // For other request types (historical, current, batch), use simple cache
      const cached = this.getFromCache(cacheKey);
      if (cached) {
        console.log(`Cache hit for ${this.getRequestDescription(request)}`);
        return cached;
      }
    }

    const providers = this.getEnabledProviders();
    if (providers.length === 0) {
      throw new Error('No stock price providers available');
    }

    let lastError: Error | undefined;

    for (const provider of providers) {
      const entry = this.providers.get(provider.name)!;

      try {
        const result = await this.fetchWithRetry(
          () => provider.fetchData(request),
          entry.config
        ) as IStockPrice | IStockPrice[];

        entry.successCount++;

        // Determine TTL based on request type
        const ttl = this.getRequestTTL(request, result);
        this.addToCache(cacheKey, result, ttl);

        console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`);
        return result;
      } catch (error) {
        entry.errorCount++;
        entry.lastError = error as Error;
        entry.lastErrorTime = new Date();
        lastError = error as Error;

        console.warn(
          `Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}`
        );
      }
    }

    throw new Error(
      `Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}`
    );
  }

  /**
   * Try incremental fetch: Only fetch NEW data since last cached timestamp
   * Returns merged result if successful, null if incremental fetch not applicable
   */
  private async tryIncrementalFetch(
    request: IStockDataRequest,
    cacheKey: string
  ): Promise<IStockPrice[] | null> {
    // Only applicable for intraday requests without date filter
    if (request.type !== 'intraday' || request.date) {
      return null;
    }

    // Check if we have similar cached data (same ticker, interval, but any limit/date)
    const baseKey = `intraday:${request.ticker}:${request.interval}:latest`;
    let cachedData: IStockPrice[] | null = null;
    let matchedKey: string | null = null;

    // Find any cached intraday data for this ticker+interval
    for (const [key, entry] of this.cache.entries()) {
      if (key.startsWith(baseKey)) {
        const age = Date.now() - entry.timestamp.getTime();
        if (entry.ttl !== Infinity && age > entry.ttl) {
          continue; // Expired
        }
        cachedData = Array.isArray(entry.price) ? entry.price as IStockPrice[] : null;
        matchedKey = key;
        break;
      }
    }

    if (!cachedData || cachedData.length === 0) {
      return null; // No cached data to build on
    }

    // Find latest timestamp in cached data
    const latestCached = cachedData.reduce((latest, price) => {
      return price.timestamp > latest ? price.timestamp : latest;
    }, new Date(0));

    // Freshness check: If latest data is less than 1 minute old, just return cache
    const dataAge = Date.now() - latestCached.getTime();
    const freshnessThreshold = 60 * 1000; // 1 minute

    if (dataAge < freshnessThreshold) {
      console.log(`🔄 Incremental cache: Latest data is ${Math.round(dataAge / 1000)}s old (< 1min), returning cached data`);
      return cachedData;
    }

    console.log(`🔄 Incremental cache: Found ${cachedData.length} cached records, latest: ${latestCached.toISOString()} (${Math.round(dataAge / 1000)}s old)`);

    // Fetch only NEW data since latest cached timestamp
    // Create a modified request with date filter
    const modifiedRequest: IStockIntradayRequest = {
      ...request,
      date: latestCached // Fetch from this date forward
    };

    const providers = this.getEnabledProviders();
    for (const provider of providers) {
      const entry = this.providers.get(provider.name)!;

      try {
        const newData = await this.fetchWithRetry(
          () => provider.fetchData(modifiedRequest),
          entry.config
        ) as IStockPrice[];

        entry.successCount++;

        // Filter out data at or before latest cached timestamp (avoid duplicates)
        const filteredNew = newData.filter(p => p.timestamp > latestCached);

        if (filteredNew.length === 0) {
          console.log(`🔄 Incremental cache: No new data since ${latestCached.toISOString()}, using cache`);
          return cachedData;
        }

        console.log(`🔄 Incremental cache: Fetched ${filteredNew.length} new records since ${latestCached.toISOString()}`);

        // Merge cached + new data
        const merged = [...cachedData, ...filteredNew];

        // Sort by timestamp (ascending)
        merged.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());

        // Deduplicate by timestamp (keep latest)
        const deduped = this.deduplicateByTimestamp(merged);

        // Apply limit if specified in original request
        const effectiveLimit = request.limit || deduped.length;
        const result = deduped.slice(-effectiveLimit); // Take most recent N

        // Update cache with merged result
        const ttl = this.getRequestTTL(request, result);
        this.addToCache(cacheKey, result, ttl);

        console.log(`🔄 Incremental cache: Returning ${result.length} total records (${cachedData.length} cached + ${filteredNew.length} new)`);
        return result;

      } catch (error) {
        entry.errorCount++;
        entry.lastError = error as Error;
        entry.lastErrorTime = new Date();
        console.warn(`Incremental fetch failed for ${provider.name}, falling back to full fetch`);
        continue; // Try next provider or fall back to normal fetch
      }
    }

    return null; // Incremental fetch failed, fall back to normal fetch
  }

  /**
   * Deduplicate array of prices by timestamp, keeping the latest data for each timestamp
   */
  private deduplicateByTimestamp(prices: IStockPrice[]): IStockPrice[] {
    const seen = new Map<number, IStockPrice>();

    for (const price of prices) {
      const ts = price.timestamp.getTime();
      const existing = seen.get(ts);

      // Keep the entry with the latest fetchedAt (most recent data)
      if (!existing || price.fetchedAt > existing.fetchedAt) {
        seen.set(ts, price);
      }
    }

    return Array.from(seen.values());
  }

  /**
   * Get TTL based on request type and result
   */
  private getRequestTTL(request: IStockDataRequest, result: IStockPrice | IStockPrice[]): number {
    switch (request.type) {
      case 'historical':
        return Infinity; // Historical data never changes
      case 'current':
        return this.getCacheTTL('eod');
      case 'batch':
        return this.getCacheTTL('eod');
      case 'intraday':
        return this.getCacheTTL('intraday', request.interval);
      default:
        return this.cacheConfig.ttl;
    }
  }

  /**
   * Get human-readable description of request
   */
  private getRequestDescription(request: IStockDataRequest): string {
    switch (request.type) {
      case 'current':
        return `current price for ${request.ticker}${request.exchange ? ` on ${request.exchange}` : ''}`;
      case 'historical':
        return `historical prices for ${request.ticker} from ${request.from.toISOString().split('T')[0]} to ${request.to.toISOString().split('T')[0]}`;
      case 'intraday':
        return `intraday ${request.interval} prices for ${request.ticker}`;
      case 'batch':
        return `batch prices for ${request.tickers.length} tickers`;
      default:
        return 'data';
    }
  }

  public async checkProvidersHealth(): Promise<Map<string, boolean>> {
    const health = new Map<string, boolean>();
    
    for (const [name, entry] of this.providers) {
      if (!entry.config.enabled) {
        health.set(name, false);
        continue;
      }

      try {
        const isAvailable = await entry.provider.isAvailable();
        health.set(name, isAvailable);
      } catch (error) {
        health.set(name, false);
        console.error(`Health check failed for ${name}:`, error);
      }
    }

    return health;
  }

  public getProviderStats(): Map<string, { 
    successCount: number; 
    errorCount: number; 
    lastError?: string;
    lastErrorTime?: Date;
  }> {
    const stats = new Map();
    
    for (const [name, entry] of this.providers) {
      stats.set(name, {
        successCount: entry.successCount,
        errorCount: entry.errorCount,
        lastError: entry.lastError?.message,
        lastErrorTime: entry.lastErrorTime
      });
    }

    return stats;
  }

  public clearCache(): void {
    this.cache.clear();
    console.log('Cache cleared');
  }

  public setCacheTTL(ttl: number): void {
    this.cacheConfig.ttl = ttl;
    console.log(`Cache TTL set to ${ttl}ms`);
  }

  private async fetchWithRetry<T>(
    fetchFn: () => Promise<T>,
    config: IProviderConfig
  ): Promise<T> {
    const maxAttempts = config.retryAttempts || 1;
    let lastError: Error | undefined;

    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
      try {
        return await fetchFn();
      } catch (error) {
        lastError = error as Error;
        
        if (attempt < maxAttempts) {
          const delay = (config.retryDelay || 1000) * attempt;
          console.log(`Retry attempt ${attempt} after ${delay}ms`);
          await plugins.smartdelay.delayFor(delay);
        }
      }
    }

    throw lastError || new Error('Unknown error during fetch');
  }

  /**
   * New cache key generation for discriminated union requests
   */
  private getDataCacheKey(request: IStockDataRequest): string {
    switch (request.type) {
      case 'current':
        return `current:${request.ticker}${request.exchange ? `:${request.exchange}` : ''}`;
      case 'historical':
        const fromStr = request.from.toISOString().split('T')[0];
        const toStr = request.to.toISOString().split('T')[0];
        return `historical:${request.ticker}:${fromStr}:${toStr}${request.exchange ? `:${request.exchange}` : ''}`;
      case 'intraday':
        const dateStr = request.date ? request.date.toISOString().split('T')[0] : 'latest';
        const limitStr = request.limit ? `:limit${request.limit}` : '';
        return `intraday:${request.ticker}:${request.interval}:${dateStr}${limitStr}${request.exchange ? `:${request.exchange}` : ''}`;
      case 'batch':
        const tickers = request.tickers.sort().join(',');
        return `batch:${tickers}${request.exchange ? `:${request.exchange}` : ''}`;
      default:
        return `unknown:${JSON.stringify(request)}`;
    }
  }

  private getFromCache(key: string): IStockPrice | IStockPrice[] | null {
    const entry = this.cache.get(key);

    if (!entry) {
      return null;
    }

    // Check if cache entry has expired
    const age = Date.now() - entry.timestamp.getTime();
    if (entry.ttl !== Infinity && age > entry.ttl) {
      this.cache.delete(key);
      return null;
    }

    return entry.price;
  }

  private addToCache(key: string, price: IStockPrice | IStockPrice[], ttl?: number): void {
    // Deduplicate array entries by timestamp before caching
    if (Array.isArray(price)) {
      const beforeCount = price.length;
      price = this.deduplicateByTimestamp(price);
      if (price.length < beforeCount) {
        console.log(`Deduplicated ${beforeCount - price.length} duplicate timestamps in cache entry for ${key}`);
      }
    }

    // Enforce max entries limit
    if (this.cache.size >= this.cacheConfig.maxEntries) {
      // Remove oldest entry
      const oldestKey = this.cache.keys().next().value;
      if (oldestKey) {
        this.cache.delete(oldestKey);
      }
    }

    this.cache.set(key, {
      price,
      timestamp: new Date(),
      ttl: ttl || this.cacheConfig.ttl
    });
  }
}