import * as plugins from '../../plugins.js'; import type { IFundamentalsProvider, IStockFundamentals, IFundamentalsRequest, IFundamentalsCurrentRequest, IFundamentalsBatchRequest } from '../interfaces/fundamentals.js'; /** * Configuration for SEC EDGAR provider */ export interface ISecEdgarConfig { userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx") cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change) fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings) timeout?: number; // Request timeout in ms } /** * Rate limiter for SEC EDGAR API * SEC requires: 10 requests per second maximum */ class RateLimiter { private requestTimes: number[] = []; private readonly maxRequestsPerSecond = 10; public async waitForSlot(): Promise { const now = Date.now(); const oneSecondAgo = now - 1000; // Remove requests older than 1 second this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo); // If we've hit the limit, wait if (this.requestTimes.length >= this.maxRequestsPerSecond) { const oldestRequest = this.requestTimes[0]; const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer await plugins.smartdelay.delayFor(waitTime); return this.waitForSlot(); // Recursively check again } // Record this request this.requestTimes.push(now); } } /** * SEC EDGAR Fundamental Data Provider * * Features: * - Free public access (no API key required) * - All US public companies * - Financial data from 10-K/10-Q filings * - US GAAP standardized metrics * - Historical data back to ~2009 * - < 1 minute filing delay * * Documentation: https://www.sec.gov/edgar/sec-api-documentation * * Rate Limits: * - 10 requests per second (enforced by SEC) * - Requires User-Agent header in format: "Company Name Email" * * Data Sources: * - Company Facts API: /api/xbrl/companyfacts/CIK##########.json * - Ticker Lookup: /files/company_tickers.json */ export class SecEdgarProvider implements IFundamentalsProvider { public name = 'SEC EDGAR'; public priority = 100; // High priority - free, authoritative, comprehensive public readonly requiresAuth = false; // No API key needed! public readonly rateLimit = { requestsPerMinute: 600, // 10 requests/second = 600/minute requestsPerDay: undefined // No daily limit }; private logger = console; private baseUrl = 'https://data.sec.gov/api/xbrl'; private tickersUrl = 'https://www.sec.gov/files/company_tickers.json'; private userAgent: string; private config: Required; // Caching private cikCache = new Map(); private tickerListCache: { data: any; timestamp: Date } | null = null; // Rate limiting private rateLimiter = new RateLimiter(); constructor(config: ISecEdgarConfig) { // Validate User-Agent if (!config.userAgent) { throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")'); } // Validate User-Agent format (must contain space and @ symbol) if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) { throw new Error( 'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")' ); } this.userAgent = config.userAgent; this.config = { userAgent: config.userAgent, cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days timeout: config.timeout || 30000 }; } /** * Unified data fetching method */ public async fetchData( request: IFundamentalsRequest ): Promise { switch (request.type) { case 'fundamentals-current': return this.fetchFundamentals(request); case 'fundamentals-batch': return this.fetchBatchFundamentals(request); default: throw new Error(`Unsupported request type: ${(request as any).type}`); } } /** * Fetch fundamental data for a single ticker */ private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise { try { // 1. Get CIK for ticker (with caching) const cik = await this.getCIK(request.ticker); // 2. Fetch company facts from SEC (with rate limiting) const companyFacts = await this.fetchCompanyFacts(cik); // 3. Parse facts into structured fundamental data return this.parseCompanyFacts(request.ticker, cik, companyFacts); } catch (error) { this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error); throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`); } } /** * Fetch fundamental data for multiple tickers */ private async fetchBatchFundamentals( request: IFundamentalsBatchRequest ): Promise { const results: IStockFundamentals[] = []; const errors: string[] = []; for (const ticker of request.tickers) { try { const fundamentals = await this.fetchFundamentals({ type: 'fundamentals-current', ticker }); results.push(fundamentals); } catch (error) { this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error); errors.push(`${ticker}: ${error.message}`); // Continue with other tickers } } if (results.length === 0) { throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`); } return results; } /** * Get CIK (Central Index Key) for a ticker symbol * Uses SEC's public ticker-to-CIK mapping file */ private async getCIK(ticker: string): Promise { const tickerUpper = ticker.toUpperCase(); // Check cache first const cached = this.cikCache.get(tickerUpper); if (cached) { const age = Date.now() - cached.timestamp.getTime(); if (age < this.config.cikCacheTTL) { return cached.cik; } // Cache expired, remove it this.cikCache.delete(tickerUpper); } // Fetch ticker list (with caching at list level) const tickers = await this.fetchTickerList(); // Find ticker in list (case-insensitive) const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper); if (!entry) { throw new Error(`CIK not found for ticker ${ticker}`); } const cik = String((entry as any).cik_str); // Cache the result this.cikCache.set(tickerUpper, { cik, timestamp: new Date() }); return cik; } /** * Fetch the SEC ticker-to-CIK mapping list * Cached for 24 hours (list updates daily) * Uses native fetch for automatic gzip decompression */ private async fetchTickerList(): Promise { // Check cache if (this.tickerListCache) { const age = Date.now() - this.tickerListCache.timestamp.getTime(); if (age < 24 * 60 * 60 * 1000) { // 24 hours return this.tickerListCache.data; } } // Wait for rate limit slot await this.rateLimiter.waitForSlot(); // Fetch from SEC using native fetch (handles gzip automatically) const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), this.config.timeout); try { const response = await fetch(this.tickersUrl, { headers: { 'User-Agent': this.userAgent, 'Accept': 'application/json' // Note: Accept-Encoding is set automatically by fetch }, signal: controller.signal }); clearTimeout(timeoutId); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const data = await response.json(); // Cache the list this.tickerListCache = { data, timestamp: new Date() }; return data; } catch (error) { clearTimeout(timeoutId); throw error; } } /** * Fetch company facts from SEC EDGAR * Uses native fetch for automatic gzip decompression */ private async fetchCompanyFacts(cik: string): Promise { // Pad CIK to 10 digits const paddedCIK = cik.padStart(10, '0'); const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`; // Wait for rate limit slot await this.rateLimiter.waitForSlot(); // Fetch from SEC using native fetch (handles gzip automatically) const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), this.config.timeout); try { const response = await fetch(url, { headers: { 'User-Agent': this.userAgent, 'Accept': 'application/json', 'Host': 'data.sec.gov' // Note: Accept-Encoding is set automatically by fetch and gzip is handled transparently }, signal: controller.signal }); clearTimeout(timeoutId); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const data = await response.json(); // Validate response if (!data || !data.facts) { throw new Error('Invalid response from SEC EDGAR API'); } return data; } catch (error) { clearTimeout(timeoutId); throw error; } } /** * Parse SEC company facts into structured fundamental data */ private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals { const usGaap = data.facts?.['us-gaap']; if (!usGaap) { throw new Error('No US GAAP data available'); } // Extract latest values for key metrics const fundamentals: IStockFundamentals = { ticker: ticker.toUpperCase(), cik: cik, companyName: data.entityName, provider: this.name, timestamp: new Date(), fetchedAt: new Date(), // Per-share metrics earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'), earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'), sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'), weightedAverageSharesOutstanding: this.getLatestValue( usGaap, 'WeightedAverageNumberOfSharesOutstandingBasic' ), // Income statement revenue: this.getLatestValue(usGaap, 'Revenues') || this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'), netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'), operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'), grossProfit: this.getLatestValue(usGaap, 'GrossProfit'), costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'), // Balance sheet assets: this.getLatestValue(usGaap, 'Assets'), liabilities: this.getLatestValue(usGaap, 'Liabilities'), stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'), cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'), propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'), // Metadata (from latest available data point) fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy, fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp, filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed ? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed) : undefined, form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form }; return fundamentals; } /** * Get the latest value for a US GAAP metric */ private getLatestValue(usGaap: any, metricName: string): number | undefined { const metric = usGaap[metricName]; if (!metric?.units) { return undefined; } // Get the first unit type (USD, shares, etc.) const unitType = Object.keys(metric.units)[0]; const values = metric.units[unitType]; if (!values || !Array.isArray(values) || values.length === 0) { return undefined; } // Get the latest value (last in array) const latest = values[values.length - 1]; return latest?.val; } /** * Get metadata from the latest data point */ private getLatestMetadata(usGaap: any, metricName: string): any | undefined { const metric = usGaap[metricName]; if (!metric?.units) { return undefined; } const unitType = Object.keys(metric.units)[0]; const values = metric.units[unitType]; if (!values || !Array.isArray(values) || values.length === 0) { return undefined; } return values[values.length - 1]; } /** * Check if SEC EDGAR API is available * Uses native fetch for automatic gzip decompression */ public async isAvailable(): Promise { try { // Test with Apple's well-known CIK const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 5000); const response = await fetch(url, { headers: { 'User-Agent': this.userAgent, 'Accept': 'application/json' }, signal: controller.signal }); clearTimeout(timeoutId); if (!response.ok) { return false; } const data = await response.json(); return data && data.facts !== undefined; } catch (error) { this.logger.warn('SEC EDGAR provider is not available:', error); return false; } } /** * Get cache statistics */ public getCacheStats(): { cikCacheSize: number; hasTickerList: boolean; } { return { cikCacheSize: this.cikCache.size, hasTickerList: this.tickerListCache !== null }; } /** * Clear all caches */ public clearCache(): void { this.cikCache.clear(); this.tickerListCache = null; this.logger.log('SEC EDGAR cache cleared'); } }