468 lines
14 KiB
TypeScript
468 lines
14 KiB
TypeScript
import * as plugins from '../../plugins.js';
|
|
import type {
|
|
IFundamentalsProvider,
|
|
IStockFundamentals,
|
|
IFundamentalsRequest,
|
|
IFundamentalsCurrentRequest,
|
|
IFundamentalsBatchRequest
|
|
} from '../interfaces/fundamentals.js';
|
|
|
|
/**
|
|
* Configuration for SEC EDGAR provider
|
|
*/
|
|
export interface ISecEdgarConfig {
|
|
userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx")
|
|
cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change)
|
|
fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings)
|
|
timeout?: number; // Request timeout in ms
|
|
}
|
|
|
|
/**
|
|
* Rate limiter for SEC EDGAR API
|
|
* SEC requires: 10 requests per second maximum
|
|
*/
|
|
class RateLimiter {
|
|
private requestTimes: number[] = [];
|
|
private readonly maxRequestsPerSecond = 10;
|
|
|
|
public async waitForSlot(): Promise<void> {
|
|
const now = Date.now();
|
|
const oneSecondAgo = now - 1000;
|
|
|
|
// Remove requests older than 1 second
|
|
this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo);
|
|
|
|
// If we've hit the limit, wait
|
|
if (this.requestTimes.length >= this.maxRequestsPerSecond) {
|
|
const oldestRequest = this.requestTimes[0];
|
|
const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer
|
|
await plugins.smartdelay.delayFor(waitTime);
|
|
return this.waitForSlot(); // Recursively check again
|
|
}
|
|
|
|
// Record this request
|
|
this.requestTimes.push(now);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* SEC EDGAR Fundamental Data Provider
|
|
*
|
|
* Features:
|
|
* - Free public access (no API key required)
|
|
* - All US public companies
|
|
* - Financial data from 10-K/10-Q filings
|
|
* - US GAAP standardized metrics
|
|
* - Historical data back to ~2009
|
|
* - < 1 minute filing delay
|
|
*
|
|
* Documentation: https://www.sec.gov/edgar/sec-api-documentation
|
|
*
|
|
* Rate Limits:
|
|
* - 10 requests per second (enforced by SEC)
|
|
* - Requires User-Agent header in format: "Company Name Email"
|
|
*
|
|
* Data Sources:
|
|
* - Company Facts API: /api/xbrl/companyfacts/CIK##########.json
|
|
* - Ticker Lookup: /files/company_tickers.json
|
|
*/
|
|
export class SecEdgarProvider implements IFundamentalsProvider {
|
|
public name = 'SEC EDGAR';
|
|
public priority = 100; // High priority - free, authoritative, comprehensive
|
|
public readonly requiresAuth = false; // No API key needed!
|
|
public readonly rateLimit = {
|
|
requestsPerMinute: 600, // 10 requests/second = 600/minute
|
|
requestsPerDay: undefined // No daily limit
|
|
};
|
|
|
|
private logger = console;
|
|
private baseUrl = 'https://data.sec.gov/api/xbrl';
|
|
private tickersUrl = 'https://www.sec.gov/files/company_tickers.json';
|
|
private userAgent: string;
|
|
private config: Required<ISecEdgarConfig>;
|
|
|
|
// Caching
|
|
private cikCache = new Map<string, { cik: string; timestamp: Date }>();
|
|
private tickerListCache: { data: any; timestamp: Date } | null = null;
|
|
|
|
// Rate limiting
|
|
private rateLimiter = new RateLimiter();
|
|
|
|
constructor(config: ISecEdgarConfig) {
|
|
// Validate User-Agent
|
|
if (!config.userAgent) {
|
|
throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")');
|
|
}
|
|
|
|
// Validate User-Agent format (must contain space and @ symbol)
|
|
if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) {
|
|
throw new Error(
|
|
'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")'
|
|
);
|
|
}
|
|
|
|
this.userAgent = config.userAgent;
|
|
this.config = {
|
|
userAgent: config.userAgent,
|
|
cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days
|
|
fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days
|
|
timeout: config.timeout || 30000
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Unified data fetching method
|
|
*/
|
|
public async fetchData(
|
|
request: IFundamentalsRequest
|
|
): Promise<IStockFundamentals | IStockFundamentals[]> {
|
|
switch (request.type) {
|
|
case 'fundamentals-current':
|
|
return this.fetchFundamentals(request);
|
|
case 'fundamentals-batch':
|
|
return this.fetchBatchFundamentals(request);
|
|
default:
|
|
throw new Error(`Unsupported request type: ${(request as any).type}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetch fundamental data for a single ticker
|
|
*/
|
|
private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise<IStockFundamentals> {
|
|
try {
|
|
// 1. Get CIK for ticker (with caching)
|
|
const cik = await this.getCIK(request.ticker);
|
|
|
|
// 2. Fetch company facts from SEC (with rate limiting)
|
|
const companyFacts = await this.fetchCompanyFacts(cik);
|
|
|
|
// 3. Parse facts into structured fundamental data
|
|
return this.parseCompanyFacts(request.ticker, cik, companyFacts);
|
|
} catch (error) {
|
|
this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error);
|
|
throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetch fundamental data for multiple tickers
|
|
*/
|
|
private async fetchBatchFundamentals(
|
|
request: IFundamentalsBatchRequest
|
|
): Promise<IStockFundamentals[]> {
|
|
const results: IStockFundamentals[] = [];
|
|
const errors: string[] = [];
|
|
|
|
for (const ticker of request.tickers) {
|
|
try {
|
|
const fundamentals = await this.fetchFundamentals({
|
|
type: 'fundamentals-current',
|
|
ticker
|
|
});
|
|
results.push(fundamentals);
|
|
} catch (error) {
|
|
this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error);
|
|
errors.push(`${ticker}: ${error.message}`);
|
|
// Continue with other tickers
|
|
}
|
|
}
|
|
|
|
if (results.length === 0) {
|
|
throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Get CIK (Central Index Key) for a ticker symbol
|
|
* Uses SEC's public ticker-to-CIK mapping file
|
|
*/
|
|
private async getCIK(ticker: string): Promise<string> {
|
|
const tickerUpper = ticker.toUpperCase();
|
|
|
|
// Check cache first
|
|
const cached = this.cikCache.get(tickerUpper);
|
|
if (cached) {
|
|
const age = Date.now() - cached.timestamp.getTime();
|
|
if (age < this.config.cikCacheTTL) {
|
|
return cached.cik;
|
|
}
|
|
// Cache expired, remove it
|
|
this.cikCache.delete(tickerUpper);
|
|
}
|
|
|
|
// Fetch ticker list (with caching at list level)
|
|
const tickers = await this.fetchTickerList();
|
|
|
|
// Find ticker in list (case-insensitive)
|
|
const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper);
|
|
|
|
if (!entry) {
|
|
throw new Error(`CIK not found for ticker ${ticker}`);
|
|
}
|
|
|
|
const cik = String((entry as any).cik_str);
|
|
|
|
// Cache the result
|
|
this.cikCache.set(tickerUpper, {
|
|
cik,
|
|
timestamp: new Date()
|
|
});
|
|
|
|
return cik;
|
|
}
|
|
|
|
/**
|
|
* Fetch the SEC ticker-to-CIK mapping list
|
|
* Cached for 24 hours (list updates daily)
|
|
* Uses native fetch for automatic gzip decompression
|
|
*/
|
|
private async fetchTickerList(): Promise<any> {
|
|
// Check cache
|
|
if (this.tickerListCache) {
|
|
const age = Date.now() - this.tickerListCache.timestamp.getTime();
|
|
if (age < 24 * 60 * 60 * 1000) { // 24 hours
|
|
return this.tickerListCache.data;
|
|
}
|
|
}
|
|
|
|
// Wait for rate limit slot
|
|
await this.rateLimiter.waitForSlot();
|
|
|
|
// Fetch from SEC using native fetch (handles gzip automatically)
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
|
|
|
|
try {
|
|
const response = await fetch(this.tickersUrl, {
|
|
headers: {
|
|
'User-Agent': this.userAgent,
|
|
'Accept': 'application/json'
|
|
// Note: Accept-Encoding is set automatically by fetch
|
|
},
|
|
signal: controller.signal
|
|
});
|
|
|
|
clearTimeout(timeoutId);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
// Cache the list
|
|
this.tickerListCache = {
|
|
data,
|
|
timestamp: new Date()
|
|
};
|
|
|
|
return data;
|
|
} catch (error) {
|
|
clearTimeout(timeoutId);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetch company facts from SEC EDGAR
|
|
* Uses native fetch for automatic gzip decompression
|
|
*/
|
|
private async fetchCompanyFacts(cik: string): Promise<any> {
|
|
// Pad CIK to 10 digits
|
|
const paddedCIK = cik.padStart(10, '0');
|
|
const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`;
|
|
|
|
// Wait for rate limit slot
|
|
await this.rateLimiter.waitForSlot();
|
|
|
|
// Fetch from SEC using native fetch (handles gzip automatically)
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
|
|
|
|
try {
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
'User-Agent': this.userAgent,
|
|
'Accept': 'application/json',
|
|
'Host': 'data.sec.gov'
|
|
// Note: Accept-Encoding is set automatically by fetch and gzip is handled transparently
|
|
},
|
|
signal: controller.signal
|
|
});
|
|
|
|
clearTimeout(timeoutId);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
// Validate response
|
|
if (!data || !data.facts) {
|
|
throw new Error('Invalid response from SEC EDGAR API');
|
|
}
|
|
|
|
return data;
|
|
} catch (error) {
|
|
clearTimeout(timeoutId);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse SEC company facts into structured fundamental data
|
|
*/
|
|
private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals {
|
|
const usGaap = data.facts?.['us-gaap'];
|
|
|
|
if (!usGaap) {
|
|
throw new Error('No US GAAP data available');
|
|
}
|
|
|
|
// Extract latest values for key metrics
|
|
const fundamentals: IStockFundamentals = {
|
|
ticker: ticker.toUpperCase(),
|
|
cik: cik,
|
|
companyName: data.entityName,
|
|
provider: this.name,
|
|
timestamp: new Date(),
|
|
fetchedAt: new Date(),
|
|
|
|
// Per-share metrics
|
|
earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'),
|
|
earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'),
|
|
sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'),
|
|
weightedAverageSharesOutstanding: this.getLatestValue(
|
|
usGaap,
|
|
'WeightedAverageNumberOfSharesOutstandingBasic'
|
|
),
|
|
|
|
// Income statement
|
|
revenue: this.getLatestValue(usGaap, 'Revenues') ||
|
|
this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'),
|
|
netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'),
|
|
operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'),
|
|
grossProfit: this.getLatestValue(usGaap, 'GrossProfit'),
|
|
costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'),
|
|
|
|
// Balance sheet
|
|
assets: this.getLatestValue(usGaap, 'Assets'),
|
|
liabilities: this.getLatestValue(usGaap, 'Liabilities'),
|
|
stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'),
|
|
cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'),
|
|
propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'),
|
|
|
|
// Metadata (from latest available data point)
|
|
fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy,
|
|
fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp,
|
|
filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed
|
|
? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed)
|
|
: undefined,
|
|
form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form
|
|
};
|
|
|
|
return fundamentals;
|
|
}
|
|
|
|
/**
|
|
* Get the latest value for a US GAAP metric
|
|
*/
|
|
private getLatestValue(usGaap: any, metricName: string): number | undefined {
|
|
const metric = usGaap[metricName];
|
|
if (!metric?.units) {
|
|
return undefined;
|
|
}
|
|
|
|
// Get the first unit type (USD, shares, etc.)
|
|
const unitType = Object.keys(metric.units)[0];
|
|
const values = metric.units[unitType];
|
|
|
|
if (!values || !Array.isArray(values) || values.length === 0) {
|
|
return undefined;
|
|
}
|
|
|
|
// Get the latest value (last in array)
|
|
const latest = values[values.length - 1];
|
|
return latest?.val;
|
|
}
|
|
|
|
/**
|
|
* Get metadata from the latest data point
|
|
*/
|
|
private getLatestMetadata(usGaap: any, metricName: string): any | undefined {
|
|
const metric = usGaap[metricName];
|
|
if (!metric?.units) {
|
|
return undefined;
|
|
}
|
|
|
|
const unitType = Object.keys(metric.units)[0];
|
|
const values = metric.units[unitType];
|
|
|
|
if (!values || !Array.isArray(values) || values.length === 0) {
|
|
return undefined;
|
|
}
|
|
|
|
return values[values.length - 1];
|
|
}
|
|
|
|
/**
|
|
* Check if SEC EDGAR API is available
|
|
* Uses native fetch for automatic gzip decompression
|
|
*/
|
|
public async isAvailable(): Promise<boolean> {
|
|
try {
|
|
// Test with Apple's well-known CIK
|
|
const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`;
|
|
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
'User-Agent': this.userAgent,
|
|
'Accept': 'application/json'
|
|
},
|
|
signal: controller.signal
|
|
});
|
|
|
|
clearTimeout(timeoutId);
|
|
|
|
if (!response.ok) {
|
|
return false;
|
|
}
|
|
|
|
const data = await response.json();
|
|
return data && data.facts !== undefined;
|
|
} catch (error) {
|
|
this.logger.warn('SEC EDGAR provider is not available:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get cache statistics
|
|
*/
|
|
public getCacheStats(): {
|
|
cikCacheSize: number;
|
|
hasTickerList: boolean;
|
|
} {
|
|
return {
|
|
cikCacheSize: this.cikCache.size,
|
|
hasTickerList: this.tickerListCache !== null
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Clear all caches
|
|
*/
|
|
public clearCache(): void {
|
|
this.cikCache.clear();
|
|
this.tickerListCache = null;
|
|
this.logger.log('SEC EDGAR cache cleared');
|
|
}
|
|
}
|