fin.cx/opendata
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stocks): Introduce unified stock data service, new providers, improved caching and German business data tooling

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-06 22:58:46 +00:00
parent 909b30117b
commit d80bbacb08
3 changed files with 410 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
changelog.md
View File

@@ -1,5 +1,19 @@
# Changelog
## 2025-11-06 - 3.4.0 - feat(stocks)
Introduce unified stock data service, new providers, improved caching and German business data tooling
- Add StockDataService: unified API to fetch price + fundamentals with automatic enrichment and batch support
- Introduce BaseProviderService abstraction and refactor provider management, caching and retry logic
- Enhance StockPriceService: unified getData, discriminated union request types, data-type aware TTLs and smarter cache keys
- Add Marketstack provider with intraday/EOD selection, pagination, OHLCV and exchange filtering
- Add CoinGecko provider with robust rate-limiting, coin ID resolution and crypto support (current, historical, intraday)
- Add SEC EDGAR fundamentals provider: CIK lookup, company facts parsing, rate limiting and caching
- Improve FundamentalsService: unified fetching, caching and enrichment helpers (enrichWithPrice, enrichBatchWithPrices)
- Enhance Yahoo provider and other provider mappings for better company metadata and market state handling
- Add German business data tooling: JsonlDataProcessor for JSONL bulk imports, HandelsRegister browser automation with download handling and parsing
- Expose OpenData entry points: DB init, JSONL processing and Handelsregister integration; add readme/docs and usage examples
## 2025-11-02 - 3.3.0 - feat(stocks/CoinGeckoProvider)
Add CoinGecko provider for cryptocurrency prices, export and tests, and update documentation

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@fin.cx/opendata',
version: '3.3.0',
version: '3.4.0',
description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
}

473
ts/stocks/providers/provider.marketstack.ts
View File

@@ -10,32 +10,39 @@ import type {
} from '../interfaces/stockprice.js';
/**
* Marketstack API v2 Provider - Enhanced
* Marketstack API v2 Provider - Professional Plan with Intelligent Intraday Support
* Documentation: https://docs.apilayer.com/marketstack/docs/marketstack-api-v2-v-2-0-0
*
* Features:
* - Intelligent endpoint selection based on market hours
* - Real-time intraday pricing with multiple intervals (1min, 5min, 10min, 15min, 30min, 1hour)
* - End-of-Day (EOD) stock prices with historical data
* - Intraday pricing with multiple intervals (1min, 5min, 15min, 30min, 1hour)
* - Market state detection (PRE, REGULAR, POST, CLOSED)
* - Exchange filtering via MIC codes (XNAS, XNYS, XLON, etc.)
* - Supports 500,000+ tickers across 72+ exchanges worldwide
* - OHLCV data (Open, High, Low, Close, Volume)
* - Pagination for large datasets
* - Automatic fallback from intraday to EOD on errors
* - Requires API key authentication
*
* Intelligent Endpoint Selection:
* - During market hours (PRE/REGULAR/POST): Uses intraday endpoints for fresh data
* - After market close (CLOSED): Uses EOD endpoints to save API credits
* - Automatic fallback to EOD if intraday fails (rate limits, plan restrictions, etc.)
*
* Rate Limits:
* - Free Plan: 100 requests/month (EOD only)
* - Basic Plan: 10,000 requests/month
* - Professional Plan: 100,000 requests/month (intraday access)
* - Basic Plan: 10,000 requests/month (EOD only)
* - Professional Plan: 100,000 requests/month (intraday + EOD)
*
* Phase 1 Enhancements:
* - Historical data retrieval with date ranges
* - Exchange filtering
* - OHLCV data support
* - Pagination handling
* Intraday Access:
* - Intervals below 15min (1min, 5min, 10min) require Professional Plan or higher
* - Real-time data from IEX Exchange for US tickers
* - Symbol formatting: Periods replaced with hyphens for intraday (BRK.B → BRK-B)
*/
export class MarketstackProvider implements IStockProvider {
public name = 'Marketstack';
public priority = 80; // Lower than Yahoo (100) due to rate limits and EOD-only data
public priority = 90; // Increased from 80 - now supports real-time intraday data during market hours
public readonly requiresAuth = true;
public readonly rateLimit = {
requestsPerMinute: undefined, // No per-minute limit specified
@@ -73,41 +80,105 @@ export class MarketstackProvider implements IStockProvider {
}
/**
* Fetch current/latest EOD price for a single ticker (new API)
* Fetch current/latest price with intelligent endpoint selection
* Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
* Uses EOD after market close (CLOSED) to save API credits
*/
private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
try {
let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
// Determine current market state
const marketState = this.getUsMarketState();
const useIntraday = this.shouldUseIntradayEndpoint(marketState);
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
if (useIntraday) {
// Use intraday endpoint for fresh data during market hours
return await this.fetchCurrentPriceIntraday(request, marketState);
} else {
// Use EOD endpoint for after-close data
return await this.fetchCurrentPriceEod(request);
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
// For single ticker endpoint, response is direct object (not wrapped in data field)
if (!responseData || !responseData.close) {
throw new Error(`No data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData, 'eod');
} catch (error) {
this.logger.error(`Failed to fetch current price for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch current price for ${request.ticker}: ${error.message}`);
// If intraday fails, fallback to EOD with warning
if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
this.logger.warn(`Intraday endpoint failed for ${request.ticker}, falling back to EOD:`, error.message);
try {
return await this.fetchCurrentPriceEod(request);
} catch (eodError) {
// Both failed, throw original error
throw error;
}
}
throw error;
}
}
/**
* Fetch current price using intraday endpoint (during market hours)
* Uses 1min interval for most recent data
*/
private async fetchCurrentPriceIntraday(
request: IStockCurrentRequest,
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): Promise<IStockPrice> {
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday/latest?access_key=${this.apiKey}`;
url += `&interval=1min`; // Use 1min for most recent data
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData || !responseData.close) {
throw new Error(`No intraday data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData, 'intraday', marketState);
}
/**
* Fetch current price using EOD endpoint (after market close)
*/
private async fetchCurrentPriceEod(request: IStockCurrentRequest): Promise<IStockPrice> {
let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
// For single ticker endpoint, response is direct object (not wrapped in data field)
if (!responseData || !responseData.close) {
throw new Error(`No data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData, 'eod');
}
/**
* Fetch historical EOD prices for a ticker with date range
*/
@@ -179,62 +250,212 @@ export class MarketstackProvider implements IStockProvider {
}
/**
* Fetch intraday prices with specified interval (Phase 2 placeholder)
* Fetch intraday prices with specified interval
* Supports intervals: 1min, 5min, 10min, 15min, 30min, 1hour
* Note: Intervals below 15min require Professional Plan or higher
*/
private async fetchIntradayPrices(request: IStockIntradayRequest): Promise<IStockPrice[]> {
throw new Error('Intraday data support coming in Phase 2. For now, use EOD data with type: "current" or "historical"');
try {
const allPrices: IStockPrice[] = [];
let offset = 0;
const limit = 1000; // Max per page for intraday
const maxRecords = 10000; // Safety limit
// Format symbol for intraday endpoint (replace . with -)
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
while (true) {
let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday?access_key=${this.apiKey}`;
url += `&interval=${request.interval}`;
url += `&limit=${limit}`;
url += `&offset=${offset}`;
// Add date filter if specified
if (request.date) {
url += `&date_from=${this.formatDate(request.date)}`;
url += `&date_to=${this.formatDate(request.date)}`;
}
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
// Map data to stock prices
for (const data of responseData.data) {
try {
allPrices.push(this.mapToStockPrice(data, 'intraday'));
} catch (error) {
this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
}
}
// Check if we have more pages
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Honor limit from request if specified, or safety limit
if (!hasMore || (request.limit && allPrices.length >= request.limit) || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
// Apply limit if specified
if (request.limit && allPrices.length > request.limit) {
return allPrices.slice(0, request.limit);
}
return allPrices;
} catch (error) {
this.logger.error(`Failed to fetch intraday prices for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch intraday prices for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch current prices for multiple tickers (new API)
* Fetch current prices for multiple tickers with intelligent endpoint selection
* Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
* Uses EOD after market close (CLOSED) to save API credits
*/
private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
try {
const symbols = request.tickers.join(',');
let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
// Determine current market state
const marketState = this.getUsMarketState();
const useIntraday = this.shouldUseIntradayEndpoint(marketState);
if (request.exchange) {
url += `&exchange=${request.exchange}`;
if (useIntraday) {
return await this.fetchBatchCurrentPricesIntraday(request, marketState);
} else {
return await this.fetchBatchCurrentPricesEod(request);
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
const prices: IStockPrice[] = [];
for (const data of responseData.data) {
} catch (error) {
// Fallback to EOD if intraday fails
if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
this.logger.warn(`Intraday batch endpoint failed, falling back to EOD:`, error.message);
try {
prices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
// Continue processing other tickers
return await this.fetchBatchCurrentPricesEod(request);
} catch (eodError) {
// Both failed, throw original error
throw error;
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch request');
}
return prices;
} catch (error) {
this.logger.error(`Failed to fetch batch current prices:`, error);
throw new Error(`Marketstack: Failed to fetch batch current prices: ${error.message}`);
throw error;
}
}
/**
* Fetch batch current prices using intraday endpoint
*/
private async fetchBatchCurrentPricesIntraday(
request: IStockBatchCurrentRequest,
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): Promise<IStockPrice[]> {
// Format symbols for intraday (replace . with -)
const formattedSymbols = request.tickers.map(t => this.formatSymbolForIntraday(t)).join(',');
let url = `${this.baseUrl}/intraday/latest?access_key=${this.apiKey}`;
url += `&symbols=${formattedSymbols}`;
url += `&interval=1min`; // Use 1min for most recent data
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
const prices: IStockPrice[] = [];
for (const data of responseData.data) {
try {
prices.push(this.mapToStockPrice(data, 'intraday', marketState));
} catch (error) {
this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch intraday request');
}
return prices;
}
/**
* Fetch batch current prices using EOD endpoint
*/
private async fetchBatchCurrentPricesEod(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
const symbols = request.tickers.join(',');
let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
const prices: IStockPrice[] = [];
for (const data of responseData.data) {
try {
prices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
// Continue processing other tickers
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch request');
}
return prices;
}
/**
* Check if the Marketstack API is available and accessible
*/
@@ -283,8 +504,15 @@ export class MarketstackProvider implements IStockProvider {
/**
* Map Marketstack API response to IStockPrice interface
* @param data - API response data
* @param dataType - Type of data (eod, intraday, live)
* @param explicitMarketState - Override market state (used for intraday data fetched during known market hours)
*/
private mapToStockPrice(data: any, dataType: 'eod' | 'intraday' | 'live' = 'eod'): IStockPrice {
private mapToStockPrice(
data: any,
dataType: 'eod' | 'intraday' | 'live' = 'eod',
explicitMarketState?: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): IStockPrice {
if (!data.close) {
throw new Error('Missing required price data');
}
@@ -301,6 +529,23 @@ export class MarketstackProvider implements IStockProvider {
const timestamp = data.date ? new Date(data.date) : new Date();
const fetchedAt = new Date();
// Determine market state intelligently
let marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
if (explicitMarketState) {
// Use provided market state (for intraday data fetched during known market hours)
marketState = explicitMarketState;
} else if (dataType === 'eod') {
// EOD data is always for closed markets
marketState = 'CLOSED';
} else if (dataType === 'intraday') {
// For intraday data without explicit state, determine from timestamp
marketState = this.getUsMarketState(timestamp.getTime());
} else {
// Default fallback
marketState = 'CLOSED';
}
const stockPrice: IStockPrice = {
ticker: data.symbol.toUpperCase(),
price: currentPrice,
@@ -310,7 +555,7 @@ export class MarketstackProvider implements IStockProvider {
previousClose: previousClose,
timestamp: timestamp,
provider: this.name,
marketState: 'CLOSED', // EOD data is always for closed markets
marketState: marketState, // Now dynamic based on data type, timestamp, and explicit state
exchange: data.exchange,
exchangeName: data.exchange_code || data.name,
@@ -373,4 +618,76 @@ export class MarketstackProvider implements IStockProvider {
const day = String(date.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
/**
* Get US market state based on Eastern Time
* Regular hours: 9:30 AM - 4:00 PM ET
* Pre-market: 4:00 AM - 9:30 AM ET
* After-hours: 4:00 PM - 8:00 PM ET
*
* @param timestampMs - Optional timestamp in milliseconds (defaults to current time)
* @returns Market state: PRE, REGULAR, POST, or CLOSED
*/
private getUsMarketState(timestampMs?: number): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' {
const now = timestampMs ? new Date(timestampMs) : new Date();
// Convert to ET (UTC-5 or UTC-4 depending on DST)
// For simplicity, we'll use a rough approximation
// TODO: Add proper timezone library for production use
const etOffset = -5; // Standard time, adjust for DST if needed
const etTime = new Date(now.getTime() + (etOffset * 60 * 60 * 1000));
// Get day of week (0 = Sunday, 6 = Saturday)
const dayOfWeek = etTime.getUTCDay();
// Check if weekend
if (dayOfWeek === 0 || dayOfWeek === 6) {
return 'CLOSED';
}
// Get hour and minute in ET
const hours = etTime.getUTCHours();
const minutes = etTime.getUTCMinutes();
const timeInMinutes = hours * 60 + minutes;
// Define market hours in minutes
const preMarketStart = 4 * 60; // 4:00 AM
const regularMarketStart = 9 * 60 + 30; // 9:30 AM
const regularMarketEnd = 16 * 60; // 4:00 PM
const afterHoursEnd = 20 * 60; // 8:00 PM
if (timeInMinutes >= preMarketStart && timeInMinutes < regularMarketStart) {
return 'PRE';
} else if (timeInMinutes >= regularMarketStart && timeInMinutes < regularMarketEnd) {
return 'REGULAR';
} else if (timeInMinutes >= regularMarketEnd && timeInMinutes < afterHoursEnd) {
return 'POST';
} else {
return 'CLOSED';
}
}
/**
* Determine if intraday endpoint should be used based on market state
* Uses intraday for PRE, REGULAR, and POST market states
* Uses EOD for CLOSED state to save API credits
*
* @param marketState - Current market state
* @returns true if intraday endpoint should be used
*/
private shouldUseIntradayEndpoint(marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'): boolean {
return marketState !== 'CLOSED';
}
/**
* Format ticker symbol for intraday endpoints
* Marketstack intraday API requires periods to be replaced with hyphens
* Example: BRK.B → BRK-B
*
* @param symbol - Original ticker symbol
* @returns Formatted symbol for intraday endpoints
*/
private formatSymbolForIntraday(symbol: string): string {
return symbol.replace(/\./g, '-');
}
}