v3.4.0

changelog.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 2025-11-06 - 3.4.0 - feat(stocks)
+Introduce unified stock data service, new providers, improved caching and German business data tooling
+
+- Add StockDataService: unified API to fetch price + fundamentals with automatic enrichment and batch support
+- Introduce BaseProviderService abstraction and refactor provider management, caching and retry logic
+- Enhance StockPriceService: unified getData, discriminated union request types, data-type aware TTLs and smarter cache keys
+- Add Marketstack provider with intraday/EOD selection, pagination, OHLCV and exchange filtering
+- Add CoinGecko provider with robust rate-limiting, coin ID resolution and crypto support (current, historical, intraday)
+- Add SEC EDGAR fundamentals provider: CIK lookup, company facts parsing, rate limiting and caching
+- Improve FundamentalsService: unified fetching, caching and enrichment helpers (enrichWithPrice, enrichBatchWithPrices)
+- Enhance Yahoo provider and other provider mappings for better company metadata and market state handling
+- Add German business data tooling: JsonlDataProcessor for JSONL bulk imports, HandelsRegister browser automation with download handling and parsing
+- Expose OpenData entry points: DB init, JSONL processing and Handelsregister integration; add readme/docs and usage examples
+
 ## 2025-11-02 - 3.3.0 - feat(stocks/CoinGeckoProvider)
 Add CoinGecko provider for cryptocurrency prices, export and tests, and update documentation
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fin.cx/opendata",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "private": false,
   "description": "A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.",
   "main": "dist_ts/index.js",

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@fin.cx/opendata',
-  version: '3.3.0',
+  version: '3.4.0',
   description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
 }

ts/stocks/providers/provider.marketstack.ts

@@ -10,32 +10,39 @@ import type {
 } from '../interfaces/stockprice.js';
 
 /**
- * Marketstack API v2 Provider - Enhanced
+ * Marketstack API v2 Provider - Professional Plan with Intelligent Intraday Support
  * Documentation: https://docs.apilayer.com/marketstack/docs/marketstack-api-v2-v-2-0-0
  *
  * Features:
+ * - Intelligent endpoint selection based on market hours
+ * - Real-time intraday pricing with multiple intervals (1min, 5min, 10min, 15min, 30min, 1hour)
  * - End-of-Day (EOD) stock prices with historical data
- * - Intraday pricing with multiple intervals (1min, 5min, 15min, 30min, 1hour)
+ * - Market state detection (PRE, REGULAR, POST, CLOSED)
  * - Exchange filtering via MIC codes (XNAS, XNYS, XLON, etc.)
  * - Supports 500,000+ tickers across 72+ exchanges worldwide
  * - OHLCV data (Open, High, Low, Close, Volume)
  * - Pagination for large datasets
+ * - Automatic fallback from intraday to EOD on errors
  * - Requires API key authentication
  *
+ * Intelligent Endpoint Selection:
+ * - During market hours (PRE/REGULAR/POST): Uses intraday endpoints for fresh data
+ * - After market close (CLOSED): Uses EOD endpoints to save API credits
+ * - Automatic fallback to EOD if intraday fails (rate limits, plan restrictions, etc.)
+ *
  * Rate Limits:
  * - Free Plan: 100 requests/month (EOD only)
- * - Basic Plan: 10,000 requests/month
- * - Professional Plan: 100,000 requests/month (intraday access)
+ * - Basic Plan: 10,000 requests/month (EOD only)
+ * - Professional Plan: 100,000 requests/month (intraday + EOD)
  *
- * Phase 1 Enhancements:
- * - Historical data retrieval with date ranges
- * - Exchange filtering
- * - OHLCV data support
- * - Pagination handling
+ * Intraday Access:
+ * - Intervals below 15min (1min, 5min, 10min) require Professional Plan or higher
+ * - Real-time data from IEX Exchange for US tickers
+ * - Symbol formatting: Periods replaced with hyphens for intraday (BRK.B → BRK-B)
  */
 export class MarketstackProvider implements IStockProvider {
   public name = 'Marketstack';
-  public priority = 80; // Lower than Yahoo (100) due to rate limits and EOD-only data
+  public priority = 90; // Increased from 80 - now supports real-time intraday data during market hours
   public readonly requiresAuth = true;
   public readonly rateLimit = {
     requestsPerMinute: undefined, // No per-minute limit specified
@@ -73,41 +80,105 @@ export class MarketstackProvider implements IStockProvider {
   }
 
   /**
-   * Fetch current/latest EOD price for a single ticker (new API)
+   * Fetch current/latest price with intelligent endpoint selection
+   * Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
+   * Uses EOD after market close (CLOSED) to save API credits
    */
   private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
     try {
-      let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
+      // Determine current market state
+      const marketState = this.getUsMarketState();
+      const useIntraday = this.shouldUseIntradayEndpoint(marketState);
 
-      // Add exchange filter if specified
-      if (request.exchange) {
-        url += `&exchange=${request.exchange}`;
+      if (useIntraday) {
+        // Use intraday endpoint for fresh data during market hours
+        return await this.fetchCurrentPriceIntraday(request, marketState);
+      } else {
+        // Use EOD endpoint for after-close data
+        return await this.fetchCurrentPriceEod(request);
       }
-
-      const response = await plugins.smartrequest.SmartRequest.create()
-        .url(url)
-        .timeout(this.config?.timeout || 10000)
-        .get();
-
-      const responseData = await response.json() as any;
-
-      // Check for API errors
-      if (responseData.error) {
-        throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
-      }
-
-      // For single ticker endpoint, response is direct object (not wrapped in data field)
-      if (!responseData || !responseData.close) {
-        throw new Error(`No data found for ticker ${request.ticker}`);
-      }
-
-      return this.mapToStockPrice(responseData, 'eod');
     } catch (error) {
-      this.logger.error(`Failed to fetch current price for ${request.ticker}:`, error);
-      throw new Error(`Marketstack: Failed to fetch current price for ${request.ticker}: ${error.message}`);
+      // If intraday fails, fallback to EOD with warning
+      if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
+        this.logger.warn(`Intraday endpoint failed for ${request.ticker}, falling back to EOD:`, error.message);
+        try {
+          return await this.fetchCurrentPriceEod(request);
+        } catch (eodError) {
+          // Both failed, throw original error
+          throw error;
+        }
+      }
+      throw error;
     }
   }
 
+  /**
+   * Fetch current price using intraday endpoint (during market hours)
+   * Uses 1min interval for most recent data
+   */
+  private async fetchCurrentPriceIntraday(
+    request: IStockCurrentRequest,
+    marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
+  ): Promise<IStockPrice> {
+    const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
+    let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday/latest?access_key=${this.apiKey}`;
+    url += `&interval=1min`; // Use 1min for most recent data
+
+    // Add exchange filter if specified
+    if (request.exchange) {
+      url += `&exchange=${request.exchange}`;
+    }
+
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(url)
+      .timeout(this.config?.timeout || 10000)
+      .get();
+
+    const responseData = await response.json() as any;
+
+    // Check for API errors
+    if (responseData.error) {
+      throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
+    }
+
+    if (!responseData || !responseData.close) {
+      throw new Error(`No intraday data found for ticker ${request.ticker}`);
+    }
+
+    return this.mapToStockPrice(responseData, 'intraday', marketState);
+  }
+
+  /**
+   * Fetch current price using EOD endpoint (after market close)
+   */
+  private async fetchCurrentPriceEod(request: IStockCurrentRequest): Promise<IStockPrice> {
+    let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
+
+    // Add exchange filter if specified
+    if (request.exchange) {
+      url += `&exchange=${request.exchange}`;
+    }
+
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(url)
+      .timeout(this.config?.timeout || 10000)
+      .get();
+
+    const responseData = await response.json() as any;
+
+    // Check for API errors
+    if (responseData.error) {
+      throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
+    }
+
+    // For single ticker endpoint, response is direct object (not wrapped in data field)
+    if (!responseData || !responseData.close) {
+      throw new Error(`No data found for ticker ${request.ticker}`);
+    }
+
+    return this.mapToStockPrice(responseData, 'eod');
+  }
+
   /**
    * Fetch historical EOD prices for a ticker with date range
    */
@@ -179,62 +250,212 @@ export class MarketstackProvider implements IStockProvider {
   }
 
   /**
-   * Fetch intraday prices with specified interval (Phase 2 placeholder)
+   * Fetch intraday prices with specified interval
+   * Supports intervals: 1min, 5min, 10min, 15min, 30min, 1hour
+   * Note: Intervals below 15min require Professional Plan or higher
    */
   private async fetchIntradayPrices(request: IStockIntradayRequest): Promise<IStockPrice[]> {
-    throw new Error('Intraday data support coming in Phase 2. For now, use EOD data with type: "current" or "historical"');
+    try {
+      const allPrices: IStockPrice[] = [];
+      let offset = 0;
+      const limit = 1000; // Max per page for intraday
+      const maxRecords = 10000; // Safety limit
+
+      // Format symbol for intraday endpoint (replace . with -)
+      const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
+
+      while (true) {
+        let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday?access_key=${this.apiKey}`;
+        url += `&interval=${request.interval}`;
+        url += `&limit=${limit}`;
+        url += `&offset=${offset}`;
+
+        // Add date filter if specified
+        if (request.date) {
+          url += `&date_from=${this.formatDate(request.date)}`;
+          url += `&date_to=${this.formatDate(request.date)}`;
+        }
+
+        // Add exchange filter if specified
+        if (request.exchange) {
+          url += `&exchange=${request.exchange}`;
+        }
+
+        const response = await plugins.smartrequest.SmartRequest.create()
+          .url(url)
+          .timeout(this.config?.timeout || 15000)
+          .get();
+
+        const responseData = await response.json() as any;
+
+        // Check for API errors
+        if (responseData.error) {
+          throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
+        }
+
+        if (!responseData?.data || !Array.isArray(responseData.data)) {
+          throw new Error('Invalid response format from Marketstack API');
+        }
+
+        // Map data to stock prices
+        for (const data of responseData.data) {
+          try {
+            allPrices.push(this.mapToStockPrice(data, 'intraday'));
+          } catch (error) {
+            this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
+          }
+        }
+
+        // Check if we have more pages
+        const pagination = responseData.pagination;
+        const hasMore = pagination && offset + limit < pagination.total;
+
+        // Honor limit from request if specified, or safety limit
+        if (!hasMore || (request.limit && allPrices.length >= request.limit) || allPrices.length >= maxRecords) {
+          break;
+        }
+
+        offset += limit;
+      }
+
+      // Apply limit if specified
+      if (request.limit && allPrices.length > request.limit) {
+        return allPrices.slice(0, request.limit);
+      }
+
+      return allPrices;
+    } catch (error) {
+      this.logger.error(`Failed to fetch intraday prices for ${request.ticker}:`, error);
+      throw new Error(`Marketstack: Failed to fetch intraday prices for ${request.ticker}: ${error.message}`);
+    }
   }
 
   /**
-   * Fetch current prices for multiple tickers (new API)
+   * Fetch current prices for multiple tickers with intelligent endpoint selection
+   * Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
+   * Uses EOD after market close (CLOSED) to save API credits
    */
   private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
     try {
-      const symbols = request.tickers.join(',');
-      let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
+      // Determine current market state
+      const marketState = this.getUsMarketState();
+      const useIntraday = this.shouldUseIntradayEndpoint(marketState);
 
-      if (request.exchange) {
-        url += `&exchange=${request.exchange}`;
+      if (useIntraday) {
+        return await this.fetchBatchCurrentPricesIntraday(request, marketState);
+      } else {
+        return await this.fetchBatchCurrentPricesEod(request);
       }
-
-      const response = await plugins.smartrequest.SmartRequest.create()
-        .url(url)
-        .timeout(this.config?.timeout || 15000)
-        .get();
-
-      const responseData = await response.json() as any;
-
-      // Check for API errors
-      if (responseData.error) {
-        throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
-      }
-
-      if (!responseData?.data || !Array.isArray(responseData.data)) {
-        throw new Error('Invalid response format from Marketstack API');
-      }
-
-      const prices: IStockPrice[] = [];
-
-      for (const data of responseData.data) {
+    } catch (error) {
+      // Fallback to EOD if intraday fails
+      if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
+        this.logger.warn(`Intraday batch endpoint failed, falling back to EOD:`, error.message);
         try {
-          prices.push(this.mapToStockPrice(data, 'eod'));
-        } catch (error) {
-          this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
-          // Continue processing other tickers
+          return await this.fetchBatchCurrentPricesEod(request);
+        } catch (eodError) {
+          // Both failed, throw original error
+          throw error;
         }
       }
-
-      if (prices.length === 0) {
-        throw new Error('No valid price data received from batch request');
-      }
-
-      return prices;
-    } catch (error) {
-      this.logger.error(`Failed to fetch batch current prices:`, error);
-      throw new Error(`Marketstack: Failed to fetch batch current prices: ${error.message}`);
+      throw error;
     }
   }
 
+  /**
+   * Fetch batch current prices using intraday endpoint
+   */
+  private async fetchBatchCurrentPricesIntraday(
+    request: IStockBatchCurrentRequest,
+    marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
+  ): Promise<IStockPrice[]> {
+    // Format symbols for intraday (replace . with -)
+    const formattedSymbols = request.tickers.map(t => this.formatSymbolForIntraday(t)).join(',');
+
+    let url = `${this.baseUrl}/intraday/latest?access_key=${this.apiKey}`;
+    url += `&symbols=${formattedSymbols}`;
+    url += `&interval=1min`; // Use 1min for most recent data
+
+    if (request.exchange) {
+      url += `&exchange=${request.exchange}`;
+    }
+
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(url)
+      .timeout(this.config?.timeout || 15000)
+      .get();
+
+    const responseData = await response.json() as any;
+
+    if (responseData.error) {
+      throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
+    }
+
+    if (!responseData?.data || !Array.isArray(responseData.data)) {
+      throw new Error('Invalid response format from Marketstack API');
+    }
+
+    const prices: IStockPrice[] = [];
+
+    for (const data of responseData.data) {
+      try {
+        prices.push(this.mapToStockPrice(data, 'intraday', marketState));
+      } catch (error) {
+        this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
+      }
+    }
+
+    if (prices.length === 0) {
+      throw new Error('No valid price data received from batch intraday request');
+    }
+
+    return prices;
+  }
+
+  /**
+   * Fetch batch current prices using EOD endpoint
+   */
+  private async fetchBatchCurrentPricesEod(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
+    const symbols = request.tickers.join(',');
+    let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
+
+    if (request.exchange) {
+      url += `&exchange=${request.exchange}`;
+    }
+
+    const response = await plugins.smartrequest.SmartRequest.create()
+      .url(url)
+      .timeout(this.config?.timeout || 15000)
+      .get();
+
+    const responseData = await response.json() as any;
+
+    // Check for API errors
+    if (responseData.error) {
+      throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
+    }
+
+    if (!responseData?.data || !Array.isArray(responseData.data)) {
+      throw new Error('Invalid response format from Marketstack API');
+    }
+
+    const prices: IStockPrice[] = [];
+
+    for (const data of responseData.data) {
+      try {
+        prices.push(this.mapToStockPrice(data, 'eod'));
+      } catch (error) {
+        this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
+        // Continue processing other tickers
+      }
+    }
+
+    if (prices.length === 0) {
+      throw new Error('No valid price data received from batch request');
+    }
+
+    return prices;
+  }
+
   /**
    * Check if the Marketstack API is available and accessible
    */
@@ -283,8 +504,15 @@ export class MarketstackProvider implements IStockProvider {
 
   /**
    * Map Marketstack API response to IStockPrice interface
+   * @param data - API response data
+   * @param dataType - Type of data (eod, intraday, live)
+   * @param explicitMarketState - Override market state (used for intraday data fetched during known market hours)
    */
-  private mapToStockPrice(data: any, dataType: 'eod' | 'intraday' | 'live' = 'eod'): IStockPrice {
+  private mapToStockPrice(
+    data: any,
+    dataType: 'eod' | 'intraday' | 'live' = 'eod',
+    explicitMarketState?: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
+  ): IStockPrice {
     if (!data.close) {
       throw new Error('Missing required price data');
     }
@@ -301,6 +529,23 @@ export class MarketstackProvider implements IStockProvider {
     const timestamp = data.date ? new Date(data.date) : new Date();
     const fetchedAt = new Date();
 
+    // Determine market state intelligently
+    let marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
+
+    if (explicitMarketState) {
+      // Use provided market state (for intraday data fetched during known market hours)
+      marketState = explicitMarketState;
+    } else if (dataType === 'eod') {
+      // EOD data is always for closed markets
+      marketState = 'CLOSED';
+    } else if (dataType === 'intraday') {
+      // For intraday data without explicit state, determine from timestamp
+      marketState = this.getUsMarketState(timestamp.getTime());
+    } else {
+      // Default fallback
+      marketState = 'CLOSED';
+    }
+
     const stockPrice: IStockPrice = {
       ticker: data.symbol.toUpperCase(),
       price: currentPrice,
@@ -310,7 +555,7 @@ export class MarketstackProvider implements IStockProvider {
       previousClose: previousClose,
       timestamp: timestamp,
       provider: this.name,
-      marketState: 'CLOSED', // EOD data is always for closed markets
+      marketState: marketState, // Now dynamic based on data type, timestamp, and explicit state
       exchange: data.exchange,
       exchangeName: data.exchange_code || data.name,
 
@@ -373,4 +618,76 @@ export class MarketstackProvider implements IStockProvider {
     const day = String(date.getDate()).padStart(2, '0');
     return `${year}-${month}-${day}`;
   }
+
+  /**
+   * Get US market state based on Eastern Time
+   * Regular hours: 9:30 AM - 4:00 PM ET
+   * Pre-market: 4:00 AM - 9:30 AM ET
+   * After-hours: 4:00 PM - 8:00 PM ET
+   *
+   * @param timestampMs - Optional timestamp in milliseconds (defaults to current time)
+   * @returns Market state: PRE, REGULAR, POST, or CLOSED
+   */
+  private getUsMarketState(timestampMs?: number): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' {
+    const now = timestampMs ? new Date(timestampMs) : new Date();
+
+    // Convert to ET (UTC-5 or UTC-4 depending on DST)
+    // For simplicity, we'll use a rough approximation
+    // TODO: Add proper timezone library for production use
+    const etOffset = -5; // Standard time, adjust for DST if needed
+    const etTime = new Date(now.getTime() + (etOffset * 60 * 60 * 1000));
+
+    // Get day of week (0 = Sunday, 6 = Saturday)
+    const dayOfWeek = etTime.getUTCDay();
+
+    // Check if weekend
+    if (dayOfWeek === 0 || dayOfWeek === 6) {
+      return 'CLOSED';
+    }
+
+    // Get hour and minute in ET
+    const hours = etTime.getUTCHours();
+    const minutes = etTime.getUTCMinutes();
+    const timeInMinutes = hours * 60 + minutes;
+
+    // Define market hours in minutes
+    const preMarketStart = 4 * 60; // 4:00 AM
+    const regularMarketStart = 9 * 60 + 30; // 9:30 AM
+    const regularMarketEnd = 16 * 60; // 4:00 PM
+    const afterHoursEnd = 20 * 60; // 8:00 PM
+
+    if (timeInMinutes >= preMarketStart && timeInMinutes < regularMarketStart) {
+      return 'PRE';
+    } else if (timeInMinutes >= regularMarketStart && timeInMinutes < regularMarketEnd) {
+      return 'REGULAR';
+    } else if (timeInMinutes >= regularMarketEnd && timeInMinutes < afterHoursEnd) {
+      return 'POST';
+    } else {
+      return 'CLOSED';
+    }
+  }
+
+  /**
+   * Determine if intraday endpoint should be used based on market state
+   * Uses intraday for PRE, REGULAR, and POST market states
+   * Uses EOD for CLOSED state to save API credits
+   *
+   * @param marketState - Current market state
+   * @returns true if intraday endpoint should be used
+   */
+  private shouldUseIntradayEndpoint(marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'): boolean {
+    return marketState !== 'CLOSED';
+  }
+
+  /**
+   * Format ticker symbol for intraday endpoints
+   * Marketstack intraday API requires periods to be replaced with hyphens
+   * Example: BRK.B → BRK-B
+   *
+   * @param symbol - Original ticker symbol
+   * @returns Formatted symbol for intraday endpoints
+   */
+  private formatSymbolForIntraday(symbol: string): string {
+    return symbol.replace(/\./g, '-');
+  }
 }