fin.cx/opendata
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stocks): Add provider fetch limits, intraday incremental fetch, cache deduplication, and provider safety/warning improvements

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-07 08:05:59 +00:00
parent 27417d81bf
commit c38f895a72
9 changed files with 1573 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
ts/stocks/providers/provider.coingecko.ts
View File

@@ -378,8 +378,18 @@ export class CoinGeckoProvider implements IStockProvider {
const marketCapData = responseData.market_caps || [];
const volumeData = responseData.total_volumes || [];
// Process each data point
for (let i = 0; i < priceData.length; i++) {
// Warn if processing large amount of historical data
const maxRecords = this.config?.maxRecords || 10000;
if (priceData.length > maxRecords) {
this.logger.warn(
`Historical request for ${request.ticker} returned ${priceData.length} records, ` +
`which exceeds maxRecords limit of ${maxRecords}. Processing first ${maxRecords} only.`
);
}
// Process each data point (up to maxRecords)
const recordsToProcess = Math.min(priceData.length, maxRecords);
for (let i = 0; i < recordsToProcess; i++) {
const [timestamp, price] = priceData[i];
const date = new Date(timestamp);
@@ -480,8 +490,19 @@ export class CoinGeckoProvider implements IStockProvider {
const marketCapData = responseData.market_caps || [];
const volumeData = responseData.total_volumes || [];
// Apply limit if specified
const limit = request.limit || priceData.length;
// Apply default limit if user didn't specify one (performance optimization)
const effectiveLimit = request.limit || this.config?.defaultIntradayLimit || 1000;
// Warn if fetching large amount of data without explicit limit
if (!request.limit && priceData.length > effectiveLimit) {
this.logger.warn(
`Intraday request for ${request.ticker} returned ${priceData.length} records but no limit specified. ` +
`Applying default limit of ${effectiveLimit}. Consider adding a limit to the request for better performance.`
);
}
// Apply limit (take most recent data)
const limit = Math.min(effectiveLimit, priceData.length);
const dataToProcess = priceData.slice(-limit);
for (let i = 0; i < dataToProcess.length; i++) {
@@ -624,21 +645,34 @@ export class CoinGeckoProvider implements IStockProvider {
const coinList = await response.json() as ICoinListItem[];
// Clear cache before rebuilding to prevent memory leak
// Keep only entries that are in priorityTickerMap
const priorityEntries = new Map<string, string>();
for (const [key, value] of this.priorityTickerMap) {
priorityEntries.set(key, value);
}
this.coinMapCache.clear();
// Restore priority mappings
for (const [key, value] of priorityEntries) {
this.coinMapCache.set(key, value);
}
// Build mapping: symbol -> id
for (const coin of coinList) {
const symbol = coin.symbol.toLowerCase();
const id = coin.id.toLowerCase();
// Don't overwrite priority mappings or existing cache entries
if (!this.priorityTickerMap.has(symbol) && !this.coinMapCache.has(symbol)) {
// Don't overwrite priority mappings
if (!this.priorityTickerMap.has(symbol)) {
this.coinMapCache.set(symbol, id);
}
// Always cache the ID mapping
// Always cache the ID mapping (id -> id for when users pass CoinGecko IDs directly)
this.coinMapCache.set(id, id);
}
this.coinListLoadedAt = new Date();
this.logger.info(`Loaded ${coinList.length} coins from CoinGecko`);
this.logger.info(`Loaded ${coinList.length} coins from CoinGecko (cache: ${this.coinMapCache.size} entries)`);
} catch (error) {
this.logger.error('Failed to load coin list from CoinGecko:', error);
// Don't throw - we can still work with direct IDs

25
ts/stocks/providers/provider.marketstack.ts
View File

@@ -187,7 +187,7 @@ export class MarketstackProvider implements IStockProvider {
const allPrices: IStockPrice[] = [];
let offset = request.offset || 0;
const limit = request.limit || 1000; // Max per page
const maxRecords = 10000; // Safety limit
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
while (true) {
let url = `${this.baseUrl}/eod?access_key=${this.apiKey}`;
@@ -259,7 +259,18 @@ export class MarketstackProvider implements IStockProvider {
const allPrices: IStockPrice[] = [];
let offset = 0;
const limit = 1000; // Max per page for intraday
const maxRecords = 10000; // Safety limit
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
// Apply default limit if user didn't specify one (performance optimization)
const effectiveLimit = request.limit || this.config?.defaultIntradayLimit || 1000;
// Warn if fetching large amount of data without explicit limit
if (!request.limit && effectiveLimit > 1000) {
this.logger.warn(
`Intraday request for ${request.ticker} without explicit limit will fetch up to ${effectiveLimit} records. ` +
`Consider adding a limit to the request for better performance.`
);
}
// Format symbol for intraday endpoint (replace . with -)
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
@@ -310,17 +321,17 @@ export class MarketstackProvider implements IStockProvider {
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Honor limit from request if specified, or safety limit
if (!hasMore || (request.limit && allPrices.length >= request.limit) || allPrices.length >= maxRecords) {
// Honor effective limit or safety maxRecords
if (!hasMore || allPrices.length >= effectiveLimit || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
// Apply limit if specified
if (request.limit && allPrices.length > request.limit) {
return allPrices.slice(0, request.limit);
// Apply effective limit
if (allPrices.length > effectiveLimit) {
return allPrices.slice(0, effectiveLimit);
}
return allPrices;