fin.cx/opendata
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(stocks): Add provider fetch limits, intraday incremental fetch, cache deduplication, and provider safety/warning improvements

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-07 08:05:59 +00:00
parent 27417d81bf
commit c38f895a72
9 changed files with 1573 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
ts/stocks/providers/provider.marketstack.ts
View File

@@ -187,7 +187,7 @@ export class MarketstackProvider implements IStockProvider {
const allPrices: IStockPrice[] = [];
let offset = request.offset || 0;
const limit = request.limit || 1000; // Max per page
const maxRecords = 10000; // Safety limit
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
while (true) {
let url = `${this.baseUrl}/eod?access_key=${this.apiKey}`;
@@ -259,7 +259,18 @@ export class MarketstackProvider implements IStockProvider {
const allPrices: IStockPrice[] = [];
let offset = 0;
const limit = 1000; // Max per page for intraday
const maxRecords = 10000; // Safety limit
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
// Apply default limit if user didn't specify one (performance optimization)
const effectiveLimit = request.limit || this.config?.defaultIntradayLimit || 1000;
// Warn if fetching large amount of data without explicit limit
if (!request.limit && effectiveLimit > 1000) {
this.logger.warn(
`Intraday request for ${request.ticker} without explicit limit will fetch up to ${effectiveLimit} records. ` +
`Consider adding a limit to the request for better performance.`
);
}
// Format symbol for intraday endpoint (replace . with -)
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
@@ -310,17 +321,17 @@ export class MarketstackProvider implements IStockProvider {
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Honor limit from request if specified, or safety limit
if (!hasMore || (request.limit && allPrices.length >= request.limit) || allPrices.length >= maxRecords) {
// Honor effective limit or safety maxRecords
if (!hasMore || allPrices.length >= effectiveLimit || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
// Apply limit if specified
if (request.limit && allPrices.length > request.limit) {
return allPrices.slice(0, request.limit);
// Apply effective limit
if (allPrices.length > effectiveLimit) {
return allPrices.slice(0, effectiveLimit);
}
return allPrices;