diff --git a/changelog.md b/changelog.md index 90a2235..759a2f2 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,19 @@ # Changelog +## 2025-11-06 - 3.4.0 - feat(stocks) +Introduce unified stock data service, new providers, improved caching and German business data tooling + +- Add StockDataService: unified API to fetch price + fundamentals with automatic enrichment and batch support +- Introduce BaseProviderService abstraction and refactor provider management, caching and retry logic +- Enhance StockPriceService: unified getData, discriminated union request types, data-type aware TTLs and smarter cache keys +- Add Marketstack provider with intraday/EOD selection, pagination, OHLCV and exchange filtering +- Add CoinGecko provider with robust rate-limiting, coin ID resolution and crypto support (current, historical, intraday) +- Add SEC EDGAR fundamentals provider: CIK lookup, company facts parsing, rate limiting and caching +- Improve FundamentalsService: unified fetching, caching and enrichment helpers (enrichWithPrice, enrichBatchWithPrices) +- Enhance Yahoo provider and other provider mappings for better company metadata and market state handling +- Add German business data tooling: JsonlDataProcessor for JSONL bulk imports, HandelsRegister browser automation with download handling and parsing +- Expose OpenData entry points: DB init, JSONL processing and Handelsregister integration; add readme/docs and usage examples + ## 2025-11-02 - 3.3.0 - feat(stocks/CoinGeckoProvider) Add CoinGecko provider for cryptocurrency prices, export and tests, and update documentation diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 5ac9b76..d7d0d62 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@fin.cx/opendata', - version: '3.3.0', + version: '3.4.0', description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.' } diff --git a/ts/stocks/providers/provider.marketstack.ts b/ts/stocks/providers/provider.marketstack.ts index 7d35b93..fd4f870 100644 --- a/ts/stocks/providers/provider.marketstack.ts +++ b/ts/stocks/providers/provider.marketstack.ts @@ -10,32 +10,39 @@ import type { } from '../interfaces/stockprice.js'; /** - * Marketstack API v2 Provider - Enhanced + * Marketstack API v2 Provider - Professional Plan with Intelligent Intraday Support * Documentation: https://docs.apilayer.com/marketstack/docs/marketstack-api-v2-v-2-0-0 * * Features: + * - Intelligent endpoint selection based on market hours + * - Real-time intraday pricing with multiple intervals (1min, 5min, 10min, 15min, 30min, 1hour) * - End-of-Day (EOD) stock prices with historical data - * - Intraday pricing with multiple intervals (1min, 5min, 15min, 30min, 1hour) + * - Market state detection (PRE, REGULAR, POST, CLOSED) * - Exchange filtering via MIC codes (XNAS, XNYS, XLON, etc.) * - Supports 500,000+ tickers across 72+ exchanges worldwide * - OHLCV data (Open, High, Low, Close, Volume) * - Pagination for large datasets + * - Automatic fallback from intraday to EOD on errors * - Requires API key authentication * + * Intelligent Endpoint Selection: + * - During market hours (PRE/REGULAR/POST): Uses intraday endpoints for fresh data + * - After market close (CLOSED): Uses EOD endpoints to save API credits + * - Automatic fallback to EOD if intraday fails (rate limits, plan restrictions, etc.) + * * Rate Limits: * - Free Plan: 100 requests/month (EOD only) - * - Basic Plan: 10,000 requests/month - * - Professional Plan: 100,000 requests/month (intraday access) + * - Basic Plan: 10,000 requests/month (EOD only) + * - Professional Plan: 100,000 requests/month (intraday + EOD) * - * Phase 1 Enhancements: - * - Historical data retrieval with date ranges - * - Exchange filtering - * - OHLCV data support - * - Pagination handling + * Intraday Access: + * - Intervals below 15min (1min, 5min, 10min) require Professional Plan or higher + * - Real-time data from IEX Exchange for US tickers + * - Symbol formatting: Periods replaced with hyphens for intraday (BRK.B → BRK-B) */ export class MarketstackProvider implements IStockProvider { public name = 'Marketstack'; - public priority = 80; // Lower than Yahoo (100) due to rate limits and EOD-only data + public priority = 90; // Increased from 80 - now supports real-time intraday data during market hours public readonly requiresAuth = true; public readonly rateLimit = { requestsPerMinute: undefined, // No per-minute limit specified @@ -73,41 +80,105 @@ export class MarketstackProvider implements IStockProvider { } /** - * Fetch current/latest EOD price for a single ticker (new API) + * Fetch current/latest price with intelligent endpoint selection + * Uses intraday during market hours (PRE, REGULAR, POST) for fresh data + * Uses EOD after market close (CLOSED) to save API credits */ private async fetchCurrentPrice(request: IStockCurrentRequest): Promise { try { - let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`; + // Determine current market state + const marketState = this.getUsMarketState(); + const useIntraday = this.shouldUseIntradayEndpoint(marketState); - // Add exchange filter if specified - if (request.exchange) { - url += `&exchange=${request.exchange}`; + if (useIntraday) { + // Use intraday endpoint for fresh data during market hours + return await this.fetchCurrentPriceIntraday(request, marketState); + } else { + // Use EOD endpoint for after-close data + return await this.fetchCurrentPriceEod(request); } - - const response = await plugins.smartrequest.SmartRequest.create() - .url(url) - .timeout(this.config?.timeout || 10000) - .get(); - - const responseData = await response.json() as any; - - // Check for API errors - if (responseData.error) { - throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); - } - - // For single ticker endpoint, response is direct object (not wrapped in data field) - if (!responseData || !responseData.close) { - throw new Error(`No data found for ticker ${request.ticker}`); - } - - return this.mapToStockPrice(responseData, 'eod'); } catch (error) { - this.logger.error(`Failed to fetch current price for ${request.ticker}:`, error); - throw new Error(`Marketstack: Failed to fetch current price for ${request.ticker}: ${error.message}`); + // If intraday fails, fallback to EOD with warning + if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) { + this.logger.warn(`Intraday endpoint failed for ${request.ticker}, falling back to EOD:`, error.message); + try { + return await this.fetchCurrentPriceEod(request); + } catch (eodError) { + // Both failed, throw original error + throw error; + } + } + throw error; } } + /** + * Fetch current price using intraday endpoint (during market hours) + * Uses 1min interval for most recent data + */ + private async fetchCurrentPriceIntraday( + request: IStockCurrentRequest, + marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' + ): Promise { + const formattedSymbol = this.formatSymbolForIntraday(request.ticker); + let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday/latest?access_key=${this.apiKey}`; + url += `&interval=1min`; // Use 1min for most recent data + + // Add exchange filter if specified + if (request.exchange) { + url += `&exchange=${request.exchange}`; + } + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .timeout(this.config?.timeout || 10000) + .get(); + + const responseData = await response.json() as any; + + // Check for API errors + if (responseData.error) { + throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); + } + + if (!responseData || !responseData.close) { + throw new Error(`No intraday data found for ticker ${request.ticker}`); + } + + return this.mapToStockPrice(responseData, 'intraday', marketState); + } + + /** + * Fetch current price using EOD endpoint (after market close) + */ + private async fetchCurrentPriceEod(request: IStockCurrentRequest): Promise { + let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`; + + // Add exchange filter if specified + if (request.exchange) { + url += `&exchange=${request.exchange}`; + } + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .timeout(this.config?.timeout || 10000) + .get(); + + const responseData = await response.json() as any; + + // Check for API errors + if (responseData.error) { + throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); + } + + // For single ticker endpoint, response is direct object (not wrapped in data field) + if (!responseData || !responseData.close) { + throw new Error(`No data found for ticker ${request.ticker}`); + } + + return this.mapToStockPrice(responseData, 'eod'); + } + /** * Fetch historical EOD prices for a ticker with date range */ @@ -179,62 +250,212 @@ export class MarketstackProvider implements IStockProvider { } /** - * Fetch intraday prices with specified interval (Phase 2 placeholder) + * Fetch intraday prices with specified interval + * Supports intervals: 1min, 5min, 10min, 15min, 30min, 1hour + * Note: Intervals below 15min require Professional Plan or higher */ private async fetchIntradayPrices(request: IStockIntradayRequest): Promise { - throw new Error('Intraday data support coming in Phase 2. For now, use EOD data with type: "current" or "historical"'); + try { + const allPrices: IStockPrice[] = []; + let offset = 0; + const limit = 1000; // Max per page for intraday + const maxRecords = 10000; // Safety limit + + // Format symbol for intraday endpoint (replace . with -) + const formattedSymbol = this.formatSymbolForIntraday(request.ticker); + + while (true) { + let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday?access_key=${this.apiKey}`; + url += `&interval=${request.interval}`; + url += `&limit=${limit}`; + url += `&offset=${offset}`; + + // Add date filter if specified + if (request.date) { + url += `&date_from=${this.formatDate(request.date)}`; + url += `&date_to=${this.formatDate(request.date)}`; + } + + // Add exchange filter if specified + if (request.exchange) { + url += `&exchange=${request.exchange}`; + } + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .timeout(this.config?.timeout || 15000) + .get(); + + const responseData = await response.json() as any; + + // Check for API errors + if (responseData.error) { + throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); + } + + if (!responseData?.data || !Array.isArray(responseData.data)) { + throw new Error('Invalid response format from Marketstack API'); + } + + // Map data to stock prices + for (const data of responseData.data) { + try { + allPrices.push(this.mapToStockPrice(data, 'intraday')); + } catch (error) { + this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error); + } + } + + // Check if we have more pages + const pagination = responseData.pagination; + const hasMore = pagination && offset + limit < pagination.total; + + // Honor limit from request if specified, or safety limit + if (!hasMore || (request.limit && allPrices.length >= request.limit) || allPrices.length >= maxRecords) { + break; + } + + offset += limit; + } + + // Apply limit if specified + if (request.limit && allPrices.length > request.limit) { + return allPrices.slice(0, request.limit); + } + + return allPrices; + } catch (error) { + this.logger.error(`Failed to fetch intraday prices for ${request.ticker}:`, error); + throw new Error(`Marketstack: Failed to fetch intraday prices for ${request.ticker}: ${error.message}`); + } } /** - * Fetch current prices for multiple tickers (new API) + * Fetch current prices for multiple tickers with intelligent endpoint selection + * Uses intraday during market hours (PRE, REGULAR, POST) for fresh data + * Uses EOD after market close (CLOSED) to save API credits */ private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise { try { - const symbols = request.tickers.join(','); - let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`; + // Determine current market state + const marketState = this.getUsMarketState(); + const useIntraday = this.shouldUseIntradayEndpoint(marketState); - if (request.exchange) { - url += `&exchange=${request.exchange}`; + if (useIntraday) { + return await this.fetchBatchCurrentPricesIntraday(request, marketState); + } else { + return await this.fetchBatchCurrentPricesEod(request); } - - const response = await plugins.smartrequest.SmartRequest.create() - .url(url) - .timeout(this.config?.timeout || 15000) - .get(); - - const responseData = await response.json() as any; - - // Check for API errors - if (responseData.error) { - throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); - } - - if (!responseData?.data || !Array.isArray(responseData.data)) { - throw new Error('Invalid response format from Marketstack API'); - } - - const prices: IStockPrice[] = []; - - for (const data of responseData.data) { + } catch (error) { + // Fallback to EOD if intraday fails + if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) { + this.logger.warn(`Intraday batch endpoint failed, falling back to EOD:`, error.message); try { - prices.push(this.mapToStockPrice(data, 'eod')); - } catch (error) { - this.logger.warn(`Failed to parse data for ${data.symbol}:`, error); - // Continue processing other tickers + return await this.fetchBatchCurrentPricesEod(request); + } catch (eodError) { + // Both failed, throw original error + throw error; } } - - if (prices.length === 0) { - throw new Error('No valid price data received from batch request'); - } - - return prices; - } catch (error) { - this.logger.error(`Failed to fetch batch current prices:`, error); - throw new Error(`Marketstack: Failed to fetch batch current prices: ${error.message}`); + throw error; } } + /** + * Fetch batch current prices using intraday endpoint + */ + private async fetchBatchCurrentPricesIntraday( + request: IStockBatchCurrentRequest, + marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' + ): Promise { + // Format symbols for intraday (replace . with -) + const formattedSymbols = request.tickers.map(t => this.formatSymbolForIntraday(t)).join(','); + + let url = `${this.baseUrl}/intraday/latest?access_key=${this.apiKey}`; + url += `&symbols=${formattedSymbols}`; + url += `&interval=1min`; // Use 1min for most recent data + + if (request.exchange) { + url += `&exchange=${request.exchange}`; + } + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .timeout(this.config?.timeout || 15000) + .get(); + + const responseData = await response.json() as any; + + if (responseData.error) { + throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); + } + + if (!responseData?.data || !Array.isArray(responseData.data)) { + throw new Error('Invalid response format from Marketstack API'); + } + + const prices: IStockPrice[] = []; + + for (const data of responseData.data) { + try { + prices.push(this.mapToStockPrice(data, 'intraday', marketState)); + } catch (error) { + this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error); + } + } + + if (prices.length === 0) { + throw new Error('No valid price data received from batch intraday request'); + } + + return prices; + } + + /** + * Fetch batch current prices using EOD endpoint + */ + private async fetchBatchCurrentPricesEod(request: IStockBatchCurrentRequest): Promise { + const symbols = request.tickers.join(','); + let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`; + + if (request.exchange) { + url += `&exchange=${request.exchange}`; + } + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .timeout(this.config?.timeout || 15000) + .get(); + + const responseData = await response.json() as any; + + // Check for API errors + if (responseData.error) { + throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`); + } + + if (!responseData?.data || !Array.isArray(responseData.data)) { + throw new Error('Invalid response format from Marketstack API'); + } + + const prices: IStockPrice[] = []; + + for (const data of responseData.data) { + try { + prices.push(this.mapToStockPrice(data, 'eod')); + } catch (error) { + this.logger.warn(`Failed to parse data for ${data.symbol}:`, error); + // Continue processing other tickers + } + } + + if (prices.length === 0) { + throw new Error('No valid price data received from batch request'); + } + + return prices; + } + /** * Check if the Marketstack API is available and accessible */ @@ -283,8 +504,15 @@ export class MarketstackProvider implements IStockProvider { /** * Map Marketstack API response to IStockPrice interface + * @param data - API response data + * @param dataType - Type of data (eod, intraday, live) + * @param explicitMarketState - Override market state (used for intraday data fetched during known market hours) */ - private mapToStockPrice(data: any, dataType: 'eod' | 'intraday' | 'live' = 'eod'): IStockPrice { + private mapToStockPrice( + data: any, + dataType: 'eod' | 'intraday' | 'live' = 'eod', + explicitMarketState?: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' + ): IStockPrice { if (!data.close) { throw new Error('Missing required price data'); } @@ -301,6 +529,23 @@ export class MarketstackProvider implements IStockProvider { const timestamp = data.date ? new Date(data.date) : new Date(); const fetchedAt = new Date(); + // Determine market state intelligently + let marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'; + + if (explicitMarketState) { + // Use provided market state (for intraday data fetched during known market hours) + marketState = explicitMarketState; + } else if (dataType === 'eod') { + // EOD data is always for closed markets + marketState = 'CLOSED'; + } else if (dataType === 'intraday') { + // For intraday data without explicit state, determine from timestamp + marketState = this.getUsMarketState(timestamp.getTime()); + } else { + // Default fallback + marketState = 'CLOSED'; + } + const stockPrice: IStockPrice = { ticker: data.symbol.toUpperCase(), price: currentPrice, @@ -310,7 +555,7 @@ export class MarketstackProvider implements IStockProvider { previousClose: previousClose, timestamp: timestamp, provider: this.name, - marketState: 'CLOSED', // EOD data is always for closed markets + marketState: marketState, // Now dynamic based on data type, timestamp, and explicit state exchange: data.exchange, exchangeName: data.exchange_code || data.name, @@ -373,4 +618,76 @@ export class MarketstackProvider implements IStockProvider { const day = String(date.getDate()).padStart(2, '0'); return `${year}-${month}-${day}`; } + + /** + * Get US market state based on Eastern Time + * Regular hours: 9:30 AM - 4:00 PM ET + * Pre-market: 4:00 AM - 9:30 AM ET + * After-hours: 4:00 PM - 8:00 PM ET + * + * @param timestampMs - Optional timestamp in milliseconds (defaults to current time) + * @returns Market state: PRE, REGULAR, POST, or CLOSED + */ + private getUsMarketState(timestampMs?: number): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' { + const now = timestampMs ? new Date(timestampMs) : new Date(); + + // Convert to ET (UTC-5 or UTC-4 depending on DST) + // For simplicity, we'll use a rough approximation + // TODO: Add proper timezone library for production use + const etOffset = -5; // Standard time, adjust for DST if needed + const etTime = new Date(now.getTime() + (etOffset * 60 * 60 * 1000)); + + // Get day of week (0 = Sunday, 6 = Saturday) + const dayOfWeek = etTime.getUTCDay(); + + // Check if weekend + if (dayOfWeek === 0 || dayOfWeek === 6) { + return 'CLOSED'; + } + + // Get hour and minute in ET + const hours = etTime.getUTCHours(); + const minutes = etTime.getUTCMinutes(); + const timeInMinutes = hours * 60 + minutes; + + // Define market hours in minutes + const preMarketStart = 4 * 60; // 4:00 AM + const regularMarketStart = 9 * 60 + 30; // 9:30 AM + const regularMarketEnd = 16 * 60; // 4:00 PM + const afterHoursEnd = 20 * 60; // 8:00 PM + + if (timeInMinutes >= preMarketStart && timeInMinutes < regularMarketStart) { + return 'PRE'; + } else if (timeInMinutes >= regularMarketStart && timeInMinutes < regularMarketEnd) { + return 'REGULAR'; + } else if (timeInMinutes >= regularMarketEnd && timeInMinutes < afterHoursEnd) { + return 'POST'; + } else { + return 'CLOSED'; + } + } + + /** + * Determine if intraday endpoint should be used based on market state + * Uses intraday for PRE, REGULAR, and POST market states + * Uses EOD for CLOSED state to save API credits + * + * @param marketState - Current market state + * @returns true if intraday endpoint should be used + */ + private shouldUseIntradayEndpoint(marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'): boolean { + return marketState !== 'CLOSED'; + } + + /** + * Format ticker symbol for intraday endpoints + * Marketstack intraday API requires periods to be replaced with hyphens + * Example: BRK.B → BRK-B + * + * @param symbol - Original ticker symbol + * @returns Formatted symbol for intraday endpoints + */ + private formatSymbolForIntraday(symbol: string): string { + return symbol.replace(/\./g, '-'); + } }