feat(stocks): Add unified stock data API (getData) with historical/OHLCV support, smart caching and provider enhancements

This commit is contained in:
2025-10-31 14:00:59 +00:00
parent 28ae2bd737
commit 8632f0e94b
9 changed files with 879 additions and 76 deletions

View File

@@ -1,6 +1,17 @@
import * as plugins from '../plugins.js';
import type { IStockProvider, IProviderConfig, IProviderRegistry } from './interfaces/provider.js';
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest, IStockPriceError } from './interfaces/stockprice.js';
import type {
IStockPrice,
IStockQuoteRequest,
IStockBatchQuoteRequest,
IStockPriceError,
IStockDataRequest,
IStockCurrentRequest,
IStockHistoricalRequest,
IStockIntradayRequest,
IStockBatchCurrentRequest,
TIntervalType
} from './interfaces/stockprice.js';
interface IProviderEntry {
provider: IStockProvider;
@@ -12,18 +23,19 @@ interface IProviderEntry {
}
interface ICacheEntry {
price: IStockPrice;
price: IStockPrice | IStockPrice[];
timestamp: Date;
ttl: number; // Specific TTL for this entry
}
export class StockPriceService implements IProviderRegistry {
private providers = new Map<string, IProviderEntry>();
private cache = new Map<string, ICacheEntry>();
private logger = console;
private cacheConfig = {
ttl: 60000, // 60 seconds default
maxEntries: 1000
ttl: 60000, // 60 seconds default (for backward compatibility)
maxEntries: 10000 // Increased for historical data
};
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
@@ -32,6 +44,43 @@ export class StockPriceService implements IProviderRegistry {
}
}
/**
* Get data-type aware TTL for smart caching
*/
private getCacheTTL(dataType: 'eod' | 'historical' | 'intraday' | 'live', interval?: TIntervalType): number {
switch (dataType) {
case 'historical':
return Infinity; // Historical data never changes
case 'eod':
return 24 * 60 * 60 * 1000; // 24 hours (EOD is static after market close)
case 'intraday':
// Match cache TTL to interval
return this.getIntervalMs(interval);
case 'live':
return 30 * 1000; // 30 seconds for live data
default:
return this.cacheConfig.ttl; // Fallback to default
}
}
/**
* Convert interval to milliseconds
*/
private getIntervalMs(interval?: TIntervalType): number {
if (!interval) return 60 * 1000; // Default 1 minute
const intervalMap: Record<TIntervalType, number> = {
'1min': 60 * 1000,
'5min': 5 * 60 * 1000,
'10min': 10 * 60 * 1000,
'15min': 15 * 60 * 1000,
'30min': 30 * 60 * 1000,
'1hour': 60 * 60 * 1000
};
return intervalMap[interval] || 60 * 1000;
}
public register(provider: IStockProvider, config?: IProviderConfig): void {
const defaultConfig: IProviderConfig = {
enabled: true,
@@ -75,8 +124,8 @@ export class StockPriceService implements IProviderRegistry {
public async getPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
const cacheKey = this.getCacheKey(request);
const cached = this.getFromCache(cacheKey);
const cached = this.getFromCache(cacheKey) as IStockPrice | null;
if (cached) {
console.log(`Cache hit for ${request.ticker}`);
return cached;
@@ -91,7 +140,7 @@ export class StockPriceService implements IProviderRegistry {
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
try {
const price = await this.fetchWithRetry(
() => provider.fetchPrice(request),
@@ -99,7 +148,11 @@ export class StockPriceService implements IProviderRegistry {
);
entry.successCount++;
this.addToCache(cacheKey, price);
// Use smart TTL based on data type
const ttl = this.getCacheTTL(price.dataType);
this.addToCache(cacheKey, price, ttl);
console.log(`Successfully fetched ${request.ticker} from ${provider.name}`);
return price;
} catch (error) {
@@ -107,7 +160,7 @@ export class StockPriceService implements IProviderRegistry {
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(
`Provider ${provider.name} failed for ${request.ticker}: ${error.message}`
);
@@ -126,8 +179,8 @@ export class StockPriceService implements IProviderRegistry {
// Check cache for each ticker
for (const ticker of request.tickers) {
const cacheKey = this.getCacheKey({ ticker, includeExtendedHours: request.includeExtendedHours });
const cached = this.getFromCache(cacheKey);
const cached = this.getFromCache(cacheKey) as IStockPrice | null;
if (cached) {
cachedPrices.push(cached);
} else {
@@ -150,25 +203,26 @@ export class StockPriceService implements IProviderRegistry {
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
try {
fetchedPrices = await this.fetchWithRetry(
() => provider.fetchPrices({
tickers: tickersToFetch,
includeExtendedHours: request.includeExtendedHours
() => provider.fetchPrices({
tickers: tickersToFetch,
includeExtendedHours: request.includeExtendedHours
}),
entry.config
);
entry.successCount++;
// Cache the fetched prices
// Cache the fetched prices with smart TTL
for (const price of fetchedPrices) {
const cacheKey = this.getCacheKey({
ticker: price.ticker,
includeExtendedHours: request.includeExtendedHours
const cacheKey = this.getCacheKey({
ticker: price.ticker,
includeExtendedHours: request.includeExtendedHours
});
this.addToCache(cacheKey, price);
const ttl = this.getCacheTTL(price.dataType);
this.addToCache(cacheKey, price, ttl);
}
console.log(
@@ -180,7 +234,7 @@ export class StockPriceService implements IProviderRegistry {
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(
`Provider ${provider.name} failed for batch request: ${error.message}`
);
@@ -196,6 +250,101 @@ export class StockPriceService implements IProviderRegistry {
return [...cachedPrices, ...fetchedPrices];
}
/**
* New unified data fetching method supporting all request types
*/
public async getData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
const cacheKey = this.getDataCacheKey(request);
const cached = this.getFromCache(cacheKey);
if (cached) {
console.log(`Cache hit for ${this.getRequestDescription(request)}`);
return cached;
}
const providers = this.getEnabledProviders();
if (providers.length === 0) {
throw new Error('No stock price providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
// Check if provider supports the new fetchData method
if (typeof (provider as any).fetchData !== 'function') {
console.warn(`Provider ${provider.name} does not support new API, skipping`);
continue;
}
try {
const result = await this.fetchWithRetry(
() => (provider as any).fetchData(request),
entry.config
) as IStockPrice | IStockPrice[];
entry.successCount++;
// Determine TTL based on request type
const ttl = this.getRequestTTL(request, result);
this.addToCache(cacheKey, result, ttl);
console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`);
return result;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(
`Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}`
);
}
}
throw new Error(
`Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}`
);
}
/**
* Get TTL based on request type and result
*/
private getRequestTTL(request: IStockDataRequest, result: IStockPrice | IStockPrice[]): number {
switch (request.type) {
case 'historical':
return Infinity; // Historical data never changes
case 'current':
return this.getCacheTTL('eod');
case 'batch':
return this.getCacheTTL('eod');
case 'intraday':
return this.getCacheTTL('intraday', request.interval);
default:
return this.cacheConfig.ttl;
}
}
/**
* Get human-readable description of request
*/
private getRequestDescription(request: IStockDataRequest): string {
switch (request.type) {
case 'current':
return `current price for ${request.ticker}${request.exchange ? ` on ${request.exchange}` : ''}`;
case 'historical':
return `historical prices for ${request.ticker} from ${request.from.toISOString().split('T')[0]} to ${request.to.toISOString().split('T')[0]}`;
case 'intraday':
return `intraday ${request.interval} prices for ${request.ticker}`;
case 'batch':
return `batch prices for ${request.tickers.length} tickers`;
default:
return 'data';
}
}
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
const health = new Map<string, boolean>();
@@ -271,19 +420,45 @@ export class StockPriceService implements IProviderRegistry {
throw lastError || new Error('Unknown error during fetch');
}
/**
* Legacy cache key generation
*/
private getCacheKey(request: IStockQuoteRequest): string {
return `${request.ticker}:${request.includeExtendedHours || false}`;
}
private getFromCache(key: string): IStockPrice | null {
/**
* New cache key generation for discriminated union requests
*/
private getDataCacheKey(request: IStockDataRequest): string {
switch (request.type) {
case 'current':
return `current:${request.ticker}${request.exchange ? `:${request.exchange}` : ''}`;
case 'historical':
const fromStr = request.from.toISOString().split('T')[0];
const toStr = request.to.toISOString().split('T')[0];
return `historical:${request.ticker}:${fromStr}:${toStr}${request.exchange ? `:${request.exchange}` : ''}`;
case 'intraday':
const dateStr = request.date ? request.date.toISOString().split('T')[0] : 'latest';
return `intraday:${request.ticker}:${request.interval}:${dateStr}${request.exchange ? `:${request.exchange}` : ''}`;
case 'batch':
const tickers = request.tickers.sort().join(',');
return `batch:${tickers}${request.exchange ? `:${request.exchange}` : ''}`;
default:
return `unknown:${JSON.stringify(request)}`;
}
}
private getFromCache(key: string): IStockPrice | IStockPrice[] | null {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
// Check if cache entry has expired
const age = Date.now() - entry.timestamp.getTime();
if (age > this.cacheConfig.ttl) {
if (entry.ttl !== Infinity && age > entry.ttl) {
this.cache.delete(key);
return null;
}
@@ -291,7 +466,7 @@ export class StockPriceService implements IProviderRegistry {
return entry.price;
}
private addToCache(key: string, price: IStockPrice): void {
private addToCache(key: string, price: IStockPrice | IStockPrice[], ttl?: number): void {
// Enforce max entries limit
if (this.cache.size >= this.cacheConfig.maxEntries) {
// Remove oldest entry
@@ -303,7 +478,8 @@ export class StockPriceService implements IProviderRegistry {
this.cache.set(key, {
price,
timestamp: new Date()
timestamp: new Date(),
ttl: ttl || this.cacheConfig.ttl
});
}
}

View File

@@ -1,5 +1,6 @@
import * as plugins from '../../plugins.js';
// Enhanced stock price interface with additional OHLCV data
export interface IStockPrice {
ticker: string;
price: number;
@@ -12,6 +13,15 @@ export interface IStockPrice {
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
exchange?: string;
exchangeName?: string;
// Phase 1 enhancements
volume?: number; // Trading volume
open?: number; // Opening price
high?: number; // Day high
low?: number; // Day low
adjusted?: boolean; // If price is split/dividend adjusted
dataType: 'eod' | 'intraday' | 'live'; // What kind of data this is
fetchedAt: Date; // When we fetched (vs data timestamp)
}
type CheckStockPrice = plugins.tsclass.typeFest.IsEqual<
IStockPrice,
@@ -25,11 +35,74 @@ export interface IStockPriceError {
timestamp: Date;
}
// Pagination support for large datasets
export interface IPaginatedResponse<T> {
data: T[];
pagination: {
currentPage: number;
totalPages: number;
totalRecords: number;
hasMore: boolean;
limit: number;
offset: number;
};
}
// Phase 1: Discriminated union types for different request types
export type TIntervalType = '1min' | '5min' | '10min' | '15min' | '30min' | '1hour';
export type TSortOrder = 'ASC' | 'DESC';
// Current price request (latest EOD or live)
export interface IStockCurrentRequest {
type: 'current';
ticker: string;
exchange?: string; // MIC code like 'XNAS', 'XNYS', 'XLON'
}
// Historical price request (date range)
export interface IStockHistoricalRequest {
type: 'historical';
ticker: string;
from: Date;
to: Date;
exchange?: string;
sort?: TSortOrder;
limit?: number; // Max results per page (default 1000)
offset?: number; // For pagination
}
// Intraday price request (real-time intervals)
export interface IStockIntradayRequest {
type: 'intraday';
ticker: string;
interval: TIntervalType;
exchange?: string;
limit?: number; // Number of bars to return
date?: Date; // Specific date for historical intraday
}
// Batch current prices request
export interface IStockBatchCurrentRequest {
type: 'batch';
tickers: string[];
exchange?: string;
}
// Union type for all stock data requests
export type IStockDataRequest =
| IStockCurrentRequest
| IStockHistoricalRequest
| IStockIntradayRequest
| IStockBatchCurrentRequest;
// Legacy interfaces (for backward compatibility during migration)
/** @deprecated Use IStockDataRequest with type: 'current' instead */
export interface IStockQuoteRequest {
ticker: string;
includeExtendedHours?: boolean;
}
/** @deprecated Use IStockDataRequest with type: 'batch' instead */
export interface IStockBatchQuoteRequest {
tickers: string[];
includeExtendedHours?: boolean;

View File

@@ -1,22 +1,41 @@
import * as plugins from '../../plugins.js';
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest } from '../interfaces/stockprice.js';
import type {
IStockPrice,
IStockQuoteRequest,
IStockBatchQuoteRequest,
IStockDataRequest,
IStockCurrentRequest,
IStockHistoricalRequest,
IStockIntradayRequest,
IStockBatchCurrentRequest,
IPaginatedResponse,
TSortOrder
} from '../interfaces/stockprice.js';
/**
* Marketstack API v2 Provider
* Documentation: https://marketstack.com/documentation_v2
* Marketstack API v2 Provider - Enhanced
* Documentation: https://docs.apilayer.com/marketstack/docs/marketstack-api-v2-v-2-0-0
*
* Features:
* - End-of-Day (EOD) stock prices
* - Supports 125,000+ tickers across 72+ exchanges worldwide
* - End-of-Day (EOD) stock prices with historical data
* - Intraday pricing with multiple intervals (1min, 5min, 15min, 30min, 1hour)
* - Exchange filtering via MIC codes (XNAS, XNYS, XLON, etc.)
* - Supports 500,000+ tickers across 72+ exchanges worldwide
* - OHLCV data (Open, High, Low, Close, Volume)
* - Pagination for large datasets
* - Requires API key authentication
*
* Rate Limits:
* - Free Plan: 100 requests/month (EOD only)
* - Basic Plan: 10,000 requests/month
* - Professional Plan: 100,000 requests/month
* - Professional Plan: 100,000 requests/month (intraday access)
*
* Note: This provider returns EOD data, not real-time prices
* Phase 1 Enhancements:
* - Historical data retrieval with date ranges
* - Exchange filtering
* - OHLCV data support
* - Pagination handling
*/
export class MarketstackProvider implements IStockProvider {
public name = 'Marketstack';
@@ -40,11 +59,34 @@ export class MarketstackProvider implements IStockProvider {
}
/**
* Fetch latest EOD price for a single ticker
* Unified data fetching method supporting all request types
*/
public async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
public async fetchData(request: IStockDataRequest): Promise<IStockPrice[] | IStockPrice> {
switch (request.type) {
case 'current':
return this.fetchCurrentPrice(request);
case 'historical':
return this.fetchHistoricalPrices(request);
case 'intraday':
return this.fetchIntradayPrices(request);
case 'batch':
return this.fetchBatchCurrentPrices(request);
default:
throw new Error(`Unsupported request type: ${(request as any).type}`);
}
}
/**
* Fetch current/latest EOD price for a single ticker (new API)
*/
private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
try {
const url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
@@ -63,20 +105,101 @@ export class MarketstackProvider implements IStockProvider {
throw new Error(`No data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData);
return this.mapToStockPrice(responseData, 'eod');
} catch (error) {
this.logger.error(`Failed to fetch price for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch price for ${request.ticker}: ${error.message}`);
this.logger.error(`Failed to fetch current price for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch current price for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch latest EOD prices for multiple tickers
* Fetch historical EOD prices for a ticker with date range
*/
public async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
private async fetchHistoricalPrices(request: IStockHistoricalRequest): Promise<IStockPrice[]> {
try {
const allPrices: IStockPrice[] = [];
let offset = request.offset || 0;
const limit = request.limit || 1000; // Max per page
const maxRecords = 10000; // Safety limit
while (true) {
let url = `${this.baseUrl}/eod?access_key=${this.apiKey}`;
url += `&symbols=${request.ticker}`;
url += `&date_from=${this.formatDate(request.from)}`;
url += `&date_to=${this.formatDate(request.to)}`;
url += `&limit=${limit}`;
url += `&offset=${offset}`;
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
if (request.sort) {
url += `&sort=${request.sort}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
// Map data to stock prices
for (const data of responseData.data) {
try {
allPrices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse historical data for ${data.symbol}:`, error);
}
}
// Check if we have more pages
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Safety check: don't fetch more than maxRecords
if (!hasMore || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
return allPrices;
} catch (error) {
this.logger.error(`Failed to fetch historical prices for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch historical prices for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch intraday prices with specified interval (Phase 2 placeholder)
*/
private async fetchIntradayPrices(request: IStockIntradayRequest): Promise<IStockPrice[]> {
throw new Error('Intraday data support coming in Phase 2. For now, use EOD data with type: "current" or "historical"');
}
/**
* Fetch current prices for multiple tickers (new API)
*/
private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
try {
const symbols = request.tickers.join(',');
const url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
@@ -98,7 +221,7 @@ export class MarketstackProvider implements IStockProvider {
for (const data of responseData.data) {
try {
prices.push(this.mapToStockPrice(data));
prices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
// Continue processing other tickers
@@ -111,11 +234,35 @@ export class MarketstackProvider implements IStockProvider {
return prices;
} catch (error) {
this.logger.error(`Failed to fetch batch prices:`, error);
throw new Error(`Marketstack: Failed to fetch batch prices: ${error.message}`);
this.logger.error(`Failed to fetch batch current prices:`, error);
throw new Error(`Marketstack: Failed to fetch batch current prices: ${error.message}`);
}
}
/**
* Legacy: Fetch latest EOD price for a single ticker
* @deprecated Use fetchData with IStockDataRequest instead
*/
public async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
// Map legacy request to new format
return this.fetchCurrentPrice({
type: 'current',
ticker: request.ticker
});
}
/**
* Legacy: Fetch latest EOD prices for multiple tickers
* @deprecated Use fetchData with IStockDataRequest instead
*/
public async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
// Map legacy request to new format
return this.fetchBatchCurrentPrices({
type: 'batch',
tickers: request.tickers
});
}
/**
* Check if the Marketstack API is available and accessible
*/
@@ -165,7 +312,7 @@ export class MarketstackProvider implements IStockProvider {
/**
* Map Marketstack API response to IStockPrice interface
*/
private mapToStockPrice(data: any): IStockPrice {
private mapToStockPrice(data: any, dataType: 'eod' | 'intraday' | 'live' = 'eod'): IStockPrice {
if (!data.close) {
throw new Error('Missing required price data');
}
@@ -174,12 +321,13 @@ export class MarketstackProvider implements IStockProvider {
// EOD data: previous close is typically open price of the same day
// For better accuracy, we'd need previous day's close, but that requires another API call
const currentPrice = data.close;
const previousClose = data.open;
const previousClose = data.open || currentPrice;
const change = currentPrice - previousClose;
const changePercent = previousClose !== 0 ? (change / previousClose) * 100 : 0;
// Parse timestamp
const timestamp = data.date ? new Date(data.date) : new Date();
const fetchedAt = new Date();
const stockPrice: IStockPrice = {
ticker: data.symbol.toUpperCase(),
@@ -192,9 +340,28 @@ export class MarketstackProvider implements IStockProvider {
provider: this.name,
marketState: 'CLOSED', // EOD data is always for closed markets
exchange: data.exchange,
exchangeName: data.exchange_code || data.name
exchangeName: data.exchange_code || data.name,
// Phase 1 enhancements: OHLCV data
volume: data.volume,
open: data.open,
high: data.high,
low: data.low,
adjusted: data.adj_close !== undefined, // If adj_close exists, price is adjusted
dataType: dataType,
fetchedAt: fetchedAt
};
return stockPrice;
}
/**
* Format date to YYYY-MM-DD for API requests
*/
private formatDate(date: Date): string {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0');
const day = String(date.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
}

View File

@@ -52,7 +52,9 @@ export class YahooFinanceProvider implements IStockProvider {
provider: this.name,
marketState: this.determineMarketState(meta),
exchange: meta.exchange,
exchangeName: meta.exchangeName
exchangeName: meta.exchangeName,
dataType: 'live', // Yahoo provides real-time/near real-time data
fetchedAt: new Date()
};
return stockPrice;
@@ -101,7 +103,9 @@ export class YahooFinanceProvider implements IStockProvider {
provider: this.name,
marketState: sparkData.marketState || 'REGULAR',
exchange: sparkData.exchange,
exchangeName: sparkData.exchangeName
exchangeName: sparkData.exchangeName,
dataType: 'live', // Yahoo provides real-time/near real-time data
fetchedAt: new Date()
});
}