diff --git a/changelog.md b/changelog.md index f539472..514c417 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,16 @@ # Changelog +## 2025-11-01 - 3.1.0 - feat(fundamentals) +Add FundamentalsService and SEC EDGAR provider with caching, rate-limiting, tests, and docs updates + +- Introduce FundamentalsService to manage fundamentals providers, caching, retry logic and provider statistics +- Add SecEdgarProvider to fetch SEC EDGAR company facts (CIK lookup, company facts parsing) with rate limiting and local caches +- Expose fundamentals interfaces and services from ts/stocks (exports updated) +- Add comprehensive tests for FundamentalsService and SecEdgarProvider (new test files) +- Update README with new Fundamentals module documentation, usage examples, and configuration guidance +- Implement caching and TTL handling for fundamentals data and provider-specific cache TTL support +- Add .claude/settings.local.json (local permissions) and various test improvements + ## 2025-10-31 - 3.0.0 - BREAKING CHANGE(stocks) Unify stock provider API to discriminated IStockDataRequest and add company name/fullname enrichment diff --git a/readme.md b/readme.md index 69164a6..93edb95 100644 --- a/readme.md +++ b/readme.md @@ -1,64 +1,8 @@ # @fin.cx/opendata -šŸš€ **Real-time financial data and German business intelligence in one powerful TypeScript library** +šŸš€ **Complete financial intelligence toolkit for TypeScript** -Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API. - -## āš ļø Breaking Changes - -### v2.1 - Enhanced Stock Market API (Current) - -**The stock market API has been significantly enhanced with a new unified request system.** - -**What Changed:** -- New discriminated union request types (`IStockDataRequest`) -- Enhanced `IStockPrice` interface with OHLCV data and metadata -- New `getData()` method replaces legacy `getPrice()` and `getPrices()` -- Historical data support with date ranges -- Exchange filtering via MIC codes -- Smart caching with data-type aware TTL - -**Migration:** -```typescript -// OLD (v2.0 and earlier) -const price = await service.getPrice({ ticker: 'AAPL' }); -const prices = await service.getPrices({ tickers: ['AAPL', 'MSFT'] }); - -// NEW (v2.1+) - Unified API -const price = await service.getData({ type: 'current', ticker: 'AAPL' }); -const prices = await service.getData({ type: 'batch', tickers: ['AAPL', 'MSFT'] }); - -// NEW - Historical data -const history = await service.getData({ - type: 'historical', - ticker: 'AAPL', - from: new Date('2024-01-01'), - to: new Date('2024-12-31') -}); - -// NEW - Exchange filtering -const price = await service.getData({ - type: 'current', - ticker: 'VOD', - exchange: 'XLON' // London Stock Exchange -}); -``` - -**Note:** Legacy `getPrice()` and `getPrices()` methods still work but are deprecated. - -### v2.0 - Directory Configuration - -**Directory paths are now MANDATORY when using German business data features.** The package no longer creates `.nogit/` directories automatically. You must explicitly configure all directory paths when instantiating `OpenData`: - -```typescript -const openData = new OpenData({ - nogitDir: '/path/to/your/data', - downloadDir: '/path/to/your/data/downloads', - germanBusinessDataDir: '/path/to/your/data/germanbusinessdata' -}); -``` - -This change enables the package to work in read-only filesystems (like Deno compiled binaries) and gives you full control over where data is stored. +Access real-time stock prices, fundamental financial data, and comprehensive German company information - all through a single, unified API. ## Installation @@ -70,85 +14,75 @@ pnpm add @fin.cx/opendata ## Quick Start -### šŸ“ˆ Stock Market Data (v2.1+ Enhanced) +### šŸ“ˆ Stock Market Data -Get comprehensive market data with EOD, historical, and OHLCV data: +Get real-time prices with company information included automatically: ```typescript import { StockPriceService, MarketstackProvider } from '@fin.cx/opendata'; -// Initialize the service with smart caching +// Initialize service with smart caching const stockService = new StockPriceService({ - ttl: 60000, // Default cache TTL (historical data cached forever) - maxEntries: 10000 // Increased for historical data + ttl: 60000, // Cache TTL (historical cached forever) + maxEntries: 10000 }); -// Register Marketstack provider with API key -stockService.register(new MarketstackProvider('YOUR_API_KEY'), { - priority: 100, - retryAttempts: 3 -}); +// Register provider with API key +stockService.register(new MarketstackProvider('YOUR_API_KEY')); -// Get current price (new unified API) +// Get current price with company name (zero extra API calls!) const apple = await stockService.getData({ type: 'current', ticker: 'AAPL' }); -console.log(`Apple: $${apple.price} (${apple.changePercent.toFixed(2)}%)`); -console.log(`OHLCV: O=${apple.open} H=${apple.high} L=${apple.low} V=${apple.volume}`); -console.log(`Company: ${apple.companyName}`); // "Apple Inc" -console.log(`Full: ${apple.companyFullName}`); // "Apple Inc (NASDAQ:AAPL)" -// Get historical data (1 year of daily prices) -const history = await stockService.getData({ - type: 'historical', - ticker: 'AAPL', - from: new Date('2024-01-01'), - to: new Date('2024-12-31'), - sort: 'DESC' // Newest first -}); -console.log(`Fetched ${history.length} trading days`); - -// Exchange-specific data (London vs NYSE) -const vodLondon = await stockService.getData({ - type: 'current', - ticker: 'VOD', - exchange: 'XLON' // London Stock Exchange -}); - -const vodNYSE = await stockService.getData({ - type: 'current', - ticker: 'VOD', - exchange: 'XNYS' // New York Stock Exchange -}); - -// Batch current prices with company names -const prices = await stockService.getData({ - type: 'batch', - tickers: ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA'] -}); - -// Display with company names (automatically included - zero extra API calls!) -for (const stock of prices) { - console.log(`${stock.companyName}: $${stock.price}`); - // Output: - // Apple Inc: $271.40 - // Microsoft Corporation: $525.76 - // Alphabet Inc - Class A: $281.48 - // Amazon.com Inc: $222.86 - // Tesla Inc: $440.10 -} - -// Use companyFullName for richer context -console.log(prices[0].companyFullName); // "Apple Inc (NASDAQ:AAPL)" +console.log(`${apple.companyFullName}: $${apple.price}`); +// Output: "Apple Inc (NASDAQ:AAPL): $270.37" ``` -### šŸ¢ German Business Data +### šŸ’° Fundamental Financial Data -Access comprehensive data on German companies: +Access comprehensive financial metrics from SEC filings - completely FREE: + +```typescript +import { SecEdgarProvider, FundamentalsService } from '@fin.cx/opendata'; + +// Setup SEC EDGAR provider (no API key required!) +const secEdgar = new SecEdgarProvider({ + userAgent: 'YourCompany youremail@example.com' +}); + +const fundamentalsService = new FundamentalsService(); +fundamentalsService.register(secEdgar); + +// Fetch fundamentals for Apple +const fundamentals = await fundamentalsService.getFundamentals('AAPL'); + +console.log({ + company: fundamentals.companyName, // "Apple Inc." + eps: fundamentals.earningsPerShareDiluted, // $6.13 + revenue: fundamentals.revenue, // $385.6B + sharesOutstanding: fundamentals.sharesOutstanding // 15.3B +}); + +// Calculate market cap and P/E ratio +const price = await stockService.getData({ type: 'current', ticker: 'AAPL' }); +const enriched = await fundamentalsService.enrichWithPrice(fundamentals, price.price); + +console.log({ + marketCap: `$${(enriched.marketCap! / 1_000_000_000_000).toFixed(2)}T`, + peRatio: enriched.priceToEarnings!.toFixed(2), + pbRatio: enriched.priceToBook?.toFixed(2) +}); +// Output: { marketCap: "$2.65T", peRatio: "28.42", pbRatio: "45.12" } +``` + +### šŸ¢ German Business Intelligence + +Access comprehensive German company data: ```typescript import { OpenData } from '@fin.cx/opendata'; import * as path from 'path'; -// REQUIRED: Configure directory paths +// Configure directory paths const openData = new OpenData({ nogitDir: path.join(process.cwd(), '.nogit'), downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), @@ -156,55 +90,93 @@ const openData = new OpenData({ }); await openData.start(); -// Create a business record -const company = new openData.CBusinessRecord(); -company.data = { - name: "TechStart GmbH", - city: "Berlin", - registrationId: "HRB 123456", - // ... more fields -}; -await company.save(); +// Search for companies +const results = await openData.handelsregister.searchCompany("Siemens AG"); -// Search companies by city -const berlinCompanies = await openData.db - .collection('businessrecords') - .find({ city: "Berlin" }) - .toArray(); - -// Import bulk data from official sources -await openData.buildInitialDb(); +// Get detailed information with documents +const details = await openData.handelsregister.getSpecificCompany({ + court: "Munich", + type: "HRB", + number: "6684" +}); ``` ## Features -### šŸŽÆ Stock Market Module (v2.1 Enhanced) +### šŸ“Š Stock Market Module -- **Company Names** - Automatic company name extraction with zero extra API calls (e.g., "Apple Inc (NASDAQ:AAPL)") -- **Historical Data** - Up to 15 years of daily EOD prices with automatic pagination -- **Exchange Filtering** - Query specific exchanges via MIC codes (XNAS, XLON, XNYS, etc.) -- **OHLCV Data** - Open, High, Low, Close, Volume for comprehensive analysis +- **Real-Time Prices** - Live and EOD stock prices from Yahoo Finance and Marketstack +- **Company Names** - Automatic company name extraction (e.g., "Apple Inc (NASDAQ:AAPL)") +- **Historical Data** - Up to 15 years of daily EOD prices with pagination +- **OHLCV Data** - Open, High, Low, Close, Volume for technical analysis +- **Exchange Filtering** - Query specific exchanges via MIC codes (XNAS, XLON, XNYS) - **Smart Caching** - Data-type aware TTL (historical cached forever, EOD 24h, live 30s) -- **Marketstack API** - 500,000+ tickers across 72+ exchanges worldwide - **Batch Operations** - Fetch 100+ symbols in one request -- **Unified API** - Discriminated union types for type-safe requests -- **Extensible Providers** - Easy to add new data sources -- **Retry Logic** - Configurable attempts and delays -- **Type-Safe** - Full TypeScript support with detailed interfaces +- **Type-Safe API** - Full TypeScript support with discriminated unions +- **Multi-Provider** - Automatic fallback between providers + +### šŸ’° Fundamental Data Module + +- **SEC EDGAR Integration** - FREE fundamental data directly from SEC filings +- **Comprehensive Metrics** - EPS, Revenue, Assets, Liabilities, Cash Flow, and more +- **All US Public Companies** - Complete coverage of SEC-registered companies +- **Historical Filings** - Data back to ~2009 +- **CIK Lookup** - Automatic ticker-to-CIK mapping with smart caching +- **Calculated Ratios** - Market Cap, P/E, P/B ratios when combined with prices +- **No API Key Required** - Direct access to SEC's public API +- **Rate Limit Management** - Built-in 10 req/sec rate limiting ### šŸ‡©šŸ‡Ŗ German Business Intelligence -- **MongoDB integration** for scalable data storage -- **Bulk JSONL import** from official German data sources -- **Handelsregister automation** - automated document retrieval -- **CRUD operations** with validation -- **Streaming processing** for multi-GB datasets +- **MongoDB Integration** - Scalable data storage for millions of records +- **Bulk JSONL Import** - Process multi-GB datasets efficiently +- **Handelsregister Automation** - Automated document retrieval +- **CRUD Operations** - Full database management with validation +- **Streaming Processing** - Handle large datasets without memory issues ## Advanced Examples -### Phase 1: Historical Data Analysis +### Combined Market Analysis -Fetch and analyze historical price data: +Combine price data with fundamentals for comprehensive analysis: + +```typescript +import { StockPriceService, MarketstackProvider, SecEdgarProvider, FundamentalsService } from '@fin.cx/opendata'; + +// Setup services +const stockService = new StockPriceService({ ttl: 60000 }); +stockService.register(new MarketstackProvider('YOUR_API_KEY')); + +const fundamentalsService = new FundamentalsService(); +fundamentalsService.register(new SecEdgarProvider({ + userAgent: 'YourCompany youremail@example.com' +})); + +// Analyze multiple companies +const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN']; + +for (const ticker of tickers) { + // Get price and fundamentals in parallel + const [price, fundamentals] = await Promise.all([ + stockService.getData({ type: 'current', ticker }), + fundamentalsService.getFundamentals(ticker) + ]); + + // Calculate metrics + const enriched = await fundamentalsService.enrichWithPrice(fundamentals, price.price); + + console.log(`\n${fundamentals.companyName} (${ticker})`); + console.log(` Price: $${price.price.toFixed(2)}`); + console.log(` Market Cap: $${(enriched.marketCap! / 1e9).toFixed(2)}B`); + console.log(` P/E Ratio: ${enriched.priceToEarnings!.toFixed(2)}`); + console.log(` Revenue: $${(fundamentals.revenue! / 1e9).toFixed(2)}B`); + console.log(` EPS: $${fundamentals.earningsPerShareDiluted!.toFixed(2)}`); +} +``` + +### Historical Data Analysis + +Fetch and analyze historical price trends: ```typescript // Get 1 year of historical data @@ -213,7 +185,7 @@ const history = await stockService.getData({ ticker: 'AAPL', from: new Date('2024-01-01'), to: new Date('2024-12-31'), - sort: 'DESC' // Newest first (default) + sort: 'DESC' // Newest first }); // Calculate statistics @@ -223,21 +195,129 @@ const low52Week = Math.min(...prices); const avgPrice = prices.reduce((a, b) => a + b) / prices.length; console.log(`52-Week Analysis for AAPL:`); -console.log(`High: $${high52Week.toFixed(2)}`); -console.log(`Low: $${low52Week.toFixed(2)}`); -console.log(`Average: $${avgPrice.toFixed(2)}`); -console.log(`Days: ${history.length}`); +console.log(` High: $${high52Week.toFixed(2)}`); +console.log(` Low: $${low52Week.toFixed(2)}`); +console.log(` Average: $${avgPrice.toFixed(2)}`); +console.log(` Days: ${history.length}`); -// Calculate daily returns -for (let i = 0; i < history.length - 1; i++) { - const todayPrice = history[i].price; - const yesterdayPrice = history[i + 1].price; - const dailyReturn = ((todayPrice - yesterdayPrice) / yesterdayPrice) * 100; - console.log(`${history[i].timestamp.toISOString().split('T')[0]}: ${dailyReturn.toFixed(2)}%`); +// Calculate Simple Moving Average +const calculateSMA = (data: IStockPrice[], period: number) => { + const sma: number[] = []; + for (let i = period - 1; i < data.length; i++) { + const sum = data.slice(i - period + 1, i + 1) + .reduce((acc, p) => acc + p.price, 0); + sma.push(sum / period); + } + return sma; +}; + +const sma20 = calculateSMA(history, 20); +const sma50 = calculateSMA(history, 50); + +console.log(`\nMoving Averages:`); +console.log(` 20-day SMA: $${sma20[0].toFixed(2)}`); +console.log(` 50-day SMA: $${sma50[0].toFixed(2)}`); +``` + +### OHLCV Technical Analysis + +Use OHLCV data for technical indicators: + +```typescript +const history = await stockService.getData({ + type: 'historical', + ticker: 'TSLA', + from: new Date('2024-11-01'), + to: new Date('2024-11-30') +}); + +// Calculate daily trading range +for (const day of history) { + const range = day.high! - day.low!; + const rangePercent = (range / day.low!) * 100; + + console.log(`${day.timestamp.toISOString().split('T')[0]}:`); + console.log(` Open: $${day.open}`); + console.log(` High: $${day.high}`); + console.log(` Low: $${day.low}`); + console.log(` Close: $${day.price}`); + console.log(` Volume: ${day.volume?.toLocaleString()}`); + console.log(` Range: $${range.toFixed(2)} (${rangePercent.toFixed(2)}%)`); } ``` -### Phase 1: Exchange-Specific Trading +### Fundamental Data Screening + +Screen stocks based on fundamental metrics: + +```typescript +const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA', 'META', 'NVDA']; + +// Fetch fundamentals for all tickers +const allFundamentals = await fundamentalsService.getBatchFundamentals(tickers); + +// Get current prices +const prices = await stockService.getData({ + type: 'batch', + tickers: tickers +}); + +// Create price map +const priceMap = new Map(prices.map(p => [p.ticker, p.price])); + +// Enrich with prices +const enriched = await fundamentalsService.enrichBatchWithPrices( + allFundamentals, + priceMap +); + +// Screen for value stocks (P/E < 25, P/B < 5) +const valueStocks = enriched.filter(f => + f.priceToEarnings && f.priceToEarnings < 25 && + f.priceToBook && f.priceToBook < 5 +); + +console.log('Value Stocks:'); +valueStocks.forEach(stock => { + console.log(`\n${stock.companyName} (${stock.ticker})`); + console.log(` P/E Ratio: ${stock.priceToEarnings!.toFixed(2)}`); + console.log(` P/B Ratio: ${stock.priceToBook!.toFixed(2)}`); + console.log(` Market Cap: $${(stock.marketCap! / 1e9).toFixed(2)}B`); +}); +``` + +### Market Dashboard + +Create a comprehensive market overview: + +```typescript +const indicators = [ + { ticker: 'AAPL', name: 'Apple' }, + { ticker: 'MSFT', name: 'Microsoft' }, + { ticker: 'GOOGL', name: 'Alphabet' }, + { ticker: 'AMZN', name: 'Amazon' }, + { ticker: 'TSLA', name: 'Tesla' } +]; + +const prices = await stockService.getData({ + type: 'batch', + tickers: indicators.map(i => i.ticker) +}); + +// Display with color-coded changes +prices.forEach(price => { + const indicator = indicators.find(i => i.ticker === price.ticker); + const arrow = price.change >= 0 ? 'ā†‘' : 'ā†“'; + const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m'; + + console.log( + `${price.companyName!.padEnd(25)} $${price.price.toFixed(2).padStart(8)} ` + + `${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m` + ); +}); +``` + +### Exchange-Specific Trading Compare prices across different exchanges: @@ -266,276 +346,17 @@ for (const exchange of exchanges) { } ``` -### Phase 1: OHLCV Technical Analysis - -Use OHLCV data for technical indicators: - -```typescript -const history = await stockService.getData({ - type: 'historical', - ticker: 'TSLA', - from: new Date('2024-11-01'), - to: new Date('2024-11-30') -}); - -// Calculate daily trading range -for (const day of history) { - const range = day.high - day.low; - const rangePercent = (range / day.low) * 100; - - console.log(`${day.timestamp.toISOString().split('T')[0]}:`); - console.log(` Open: $${day.open}`); - console.log(` High: $${day.high}`); - console.log(` Low: $${day.low}`); - console.log(` Close: $${day.price}`); - console.log(` Volume: ${day.volume?.toLocaleString()}`); - console.log(` Range: $${range.toFixed(2)} (${rangePercent.toFixed(2)}%)`); -} - -// Calculate Simple Moving Average (SMA) -const calculateSMA = (data: IStockPrice[], period: number) => { - const sma: number[] = []; - for (let i = period - 1; i < data.length; i++) { - const sum = data.slice(i - period + 1, i + 1) - .reduce((acc, p) => acc + p.price, 0); - sma.push(sum / period); - } - return sma; -}; - -const sma20 = calculateSMA(history, 20); -const sma50 = calculateSMA(history, 50); - -console.log(`20-day SMA: $${sma20[0].toFixed(2)}`); -console.log(`50-day SMA: $${sma50[0].toFixed(2)}`); -``` - -### Phase 1: Smart Caching Performance - -Leverage smart caching for efficiency: - -```typescript -// Historical data is cached FOREVER (never changes) -console.time('First historical fetch'); -const history1 = await stockService.getData({ - type: 'historical', - ticker: 'AAPL', - from: new Date('2024-01-01'), - to: new Date('2024-12-31') -}); -console.timeEnd('First historical fetch'); -// Output: First historical fetch: 2341ms - -console.time('Second historical fetch (cached)'); -const history2 = await stockService.getData({ - type: 'historical', - ticker: 'AAPL', - from: new Date('2024-01-01'), - to: new Date('2024-12-31') -}); -console.timeEnd('Second historical fetch (cached)'); -// Output: Second historical fetch (cached): 2ms (1000x faster!) - -// EOD data cached for 24 hours -const currentPrice = await stockService.getData({ - type: 'current', - ticker: 'MSFT' -}); -// Subsequent calls within 24h served from cache - -// Cache statistics -console.log(`Cache size: ${stockService['cache'].size} entries`); -``` - -### Market Dashboard - -Create an EOD market overview: - -```typescript -const indicators = [ - // Indices - { ticker: '^GSPC', name: 'S&P 500' }, - { ticker: '^DJI', name: 'DOW Jones' }, - - // Tech Giants - { ticker: 'AAPL', name: 'Apple' }, - { ticker: 'MSFT', name: 'Microsoft' }, - { ticker: 'GOOGL', name: 'Alphabet' }, - { ticker: 'AMZN', name: 'Amazon' }, - { ticker: 'TSLA', name: 'Tesla' } -]; - -const prices = await stockService.getPrices({ - tickers: indicators.map(i => i.ticker) -}); - -// Display with color-coded changes -prices.forEach(price => { - const indicator = indicators.find(i => i.ticker === price.ticker); - const arrow = price.change >= 0 ? 'ā†‘' : 'ā†“'; - const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m'; - - console.log( - `${indicator.name.padEnd(15)} ${price.price.toFixed(2).padStart(10)} ` + - `${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m` - ); -}); -``` - -### Provider Health and Statistics - -Monitor your provider health and track usage: - -```typescript -// Check provider health -const health = await stockService.checkProvidersHealth(); -console.log(`Marketstack: ${health.get('Marketstack') ? 'āœ…' : 'āŒ'}`); - -// Get provider statistics -const stats = stockService.getProviderStats(); -const marketstackStats = stats.get('Marketstack'); -console.log('Marketstack Stats:', { - successCount: marketstackStats.successCount, - errorCount: marketstackStats.errorCount, - lastError: marketstackStats.lastError -}); -``` - -### Handelsregister Integration - -Automate German company data retrieval: - -```typescript -import { OpenData } from '@fin.cx/opendata'; -import * as path from 'path'; - -// Configure paths first -const openData = new OpenData({ - nogitDir: path.join(process.cwd(), '.nogit'), - downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), - germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') -}); -await openData.start(); - -// Search for a company -const results = await openData.handelsregister.searchCompany("Siemens AG"); - -// Get detailed information and documents -const details = await openData.handelsregister.getSpecificCompany({ - court: "Munich", - type: "HRB", - number: "6684" -}); - -// Downloaded files include: -// - XML data (SI files) -// - PDF documents (AD files) -for (const file of details.files) { - await file.writeToDir('./downloads'); -} -``` - -### Combined Data Analysis - -Merge financial and business data: - -```typescript -import { OpenData, StockPriceService, MarketstackProvider } from '@fin.cx/opendata'; -import * as path from 'path'; - -// Configure OpenData with paths -const openData = new OpenData({ - nogitDir: path.join(process.cwd(), '.nogit'), - downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), - germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') -}); -await openData.start(); - -// Setup stock service -const stockService = new StockPriceService({ ttl: 60000, maxEntries: 1000 }); -stockService.register(new MarketstackProvider('YOUR_API_KEY')); - -// Find all public German companies (AG) -const publicCompanies = await openData.db - .collection('businessrecords') - .find({ legalForm: 'AG' }) - .toArray(); - -// Enrich with stock data -for (const company of publicCompanies) { - try { - // Map company to ticker (custom logic needed) - const ticker = mapCompanyToTicker(company.data.name); - - if (ticker) { - const stock = await stockService.getPrice({ ticker }); - - // Add financial metrics - company.data.stockPrice = stock.price; - company.data.marketCap = stock.price * getSharesOutstanding(ticker); - company.data.priceChange = stock.changePercent; - - await company.save(); - } - } catch (error) { - // Handle missing tickers gracefully - } -} -``` - ## Configuration -### Directory Configuration (Required for German Business Data) - -**All directory paths are mandatory when using `OpenData`.** Here are examples for different environments: - -#### Development Environment -```typescript -import { OpenData } from '@fin.cx/opendata'; -import * as path from 'path'; - -const openData = new OpenData({ - nogitDir: path.join(process.cwd(), '.nogit'), - downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), - germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') -}); -``` - -#### Production Environment -```typescript -import { OpenData } from '@fin.cx/opendata'; -import * as path from 'path'; - -const openData = new OpenData({ - nogitDir: '/var/lib/myapp/data', - downloadDir: '/var/lib/myapp/data/downloads', - germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata' -}); -``` - -#### Deno Compiled Binaries (or other read-only filesystems) -```typescript -import { OpenData } from '@fin.cx/opendata'; - -// Use OS temp directory or user data directory -const dataDir = Deno.env.get('HOME') + '/.myapp/data'; - -const openData = new OpenData({ - nogitDir: dataDir, - downloadDir: dataDir + '/downloads', - germanBusinessDataDir: dataDir + '/germanbusinessdata' -}); -``` - ### Stock Service Options ```typescript const stockService = new StockPriceService({ - ttl: 60000, // Cache for 1 minute - maxEntries: 1000 // Max cached symbols + ttl: 60000, // Default cache TTL in ms + maxEntries: 10000 // Max cached entries }); -// Marketstack - EOD data, requires API key +// Marketstack - EOD data (requires API key) stockService.register(new MarketstackProvider('YOUR_API_KEY'), { enabled: true, priority: 100, @@ -543,30 +364,72 @@ stockService.register(new MarketstackProvider('YOUR_API_KEY'), { retryAttempts: 3, retryDelay: 1000 }); + +// Yahoo Finance - Real-time data (no API key) +stockService.register(new YahooFinanceProvider(), { + enabled: true, + priority: 50 +}); ``` -### MongoDB Setup +### Fundamentals Service Options -Set environment variables for German business data: +```typescript +const fundamentalsService = new FundamentalsService({ + ttl: 90 * 24 * 60 * 60 * 1000, // 90 days (quarterly refresh) + maxEntries: 10000 +}); + +// SEC EDGAR provider (FREE - no API key!) +fundamentalsService.register(new SecEdgarProvider({ + userAgent: 'YourCompany youremail@example.com', + cikCacheTTL: 30 * 24 * 60 * 60 * 1000, // 30 days + fundamentalsCacheTTL: 90 * 24 * 60 * 60 * 1000, // 90 days + timeout: 30000 +})); +``` + +### Directory Configuration (German Business Data) + +All directory paths are mandatory when using German business data features: + +```typescript +import { OpenData } from '@fin.cx/opendata'; +import * as path from 'path'; + +// Development +const openData = new OpenData({ + nogitDir: path.join(process.cwd(), '.nogit'), + downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), + germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') +}); + +// Production +const openDataProd = new OpenData({ + nogitDir: '/var/lib/myapp/data', + downloadDir: '/var/lib/myapp/data/downloads', + germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata' +}); +``` + +### Environment Variables + +Set environment variables for API keys and database: ```env +# Marketstack API (for EOD stock data) +MARKETSTACK_COM_TOKEN=your_api_key_here + +# MongoDB (for German business data) MONGODB_URL=mongodb://localhost:27017 MONGODB_NAME=opendata MONGODB_USER=myuser MONGODB_PASS=mypass ``` -### Marketstack API Key - -Get your free API key at [marketstack.com](https://marketstack.com) and set it in your environment: - -```env -MARKETSTACK_COM_TOKEN=your_api_key_here -``` - ## API Reference -### Stock Types +### Stock Price Interfaces ```typescript interface IStockPrice { @@ -581,62 +444,154 @@ interface IStockPrice { marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'; exchange?: string; exchangeName?: string; + + // OHLCV data + volume?: number; + open?: number; + high?: number; + low?: number; + adjusted?: boolean; + dataType: 'eod' | 'intraday' | 'live'; + fetchedAt: Date; + + // Company identification + companyName?: string; // "Apple Inc" + companyFullName?: string; // "Apple Inc (NASDAQ:AAPL)" +} +``` + +### Fundamental Data Interfaces + +```typescript +interface IStockFundamentals { + ticker: string; + cik: string; + companyName: string; + provider: string; + timestamp: Date; + fetchedAt: Date; + + // Per-share metrics + earningsPerShareBasic?: number; + earningsPerShareDiluted?: number; + sharesOutstanding?: number; + + // Income statement (annual USD) + revenue?: number; + netIncome?: number; + operatingIncome?: number; + grossProfit?: number; + + // Balance sheet (annual USD) + assets?: number; + liabilities?: number; + stockholdersEquity?: number; + cash?: number; + propertyPlantEquipment?: number; + + // Calculated metrics (requires price) + marketCap?: number; // price Ɨ sharesOutstanding + priceToEarnings?: number; // price / EPS + priceToBook?: number; // marketCap / stockholdersEquity + + // Metadata + fiscalYear?: string; + fiscalQuarter?: string; + filingDate?: Date; + form?: '10-K' | '10-Q' | string; } ``` ### Key Methods **StockPriceService** -- `getPrice(request)` - Single stock price with automatic provider selection -- `getPrices(request)` - Batch prices (100+ symbols in one request) +- `getData(request)` - Unified method for all stock data (current, historical, batch) +- `getPrice(request)` - Convenience method for single current price +- `getPrices(request)` - Convenience method for batch current prices - `register(provider, config)` - Add data provider with priority and retry config - `checkProvidersHealth()` - Test all providers and return health status - `getProviderStats()` - Get success/error statistics for each provider - `clearCache()` - Clear price cache - `setCacheTTL(ttl)` - Update cache TTL dynamically +**FundamentalsService** +- `getFundamentals(ticker)` - Get fundamental data for single ticker +- `getBatchFundamentals(tickers)` - Get fundamentals for multiple tickers +- `enrichWithPrice(fundamentals, price)` - Calculate market cap, P/E, P/B ratios +- `enrichBatchWithPrices(fundamentals, priceMap)` - Batch enrich with prices +- `register(provider, config)` - Add fundamentals provider +- `checkProvidersHealth()` - Test all providers +- `getProviderStats()` - Get success/error statistics +- `clearCache()` - Clear fundamentals cache + +**SecEdgarProvider** +- āœ… FREE - No API key required +- āœ… All US public companies +- āœ… Comprehensive US GAAP financial metrics +- āœ… Historical data back to ~2009 +- āœ… Direct access to SEC filings (10-K, 10-Q) +- āœ… Smart caching (CIK: 30 days, Fundamentals: 90 days) +- āœ… Rate limiting (10 requests/second) +- ā„¹ļø Requires User-Agent header in format: "Company Name Email" + **MarketstackProvider** -- āœ… End-of-Day (EOD) data -- āœ… 125,000+ tickers across 72+ exchanges worldwide -- āœ… Batch fetching support (multiple symbols in one request) -- āœ… Comprehensive data: open, high, low, close, volume, splits, dividends +- āœ… End-of-Day (EOD) stock prices +- āœ… 500,000+ tickers across 72+ exchanges worldwide +- āœ… Historical data with pagination +- āœ… Batch fetching support +- āœ… OHLCV data (Open, High, Low, Close, Volume) +- āœ… Company names included automatically - āš ļø Requires API key (free tier: 100 requests/month) -- āš ļø EOD data only (not real-time) + +**YahooFinanceProvider** +- āœ… Real-time stock prices +- āœ… No API key required +- āœ… Global coverage +- āœ… Company names included +- āš ļø Rate limits may apply **OpenData** - `start()` - Initialize MongoDB connection - `buildInitialDb()` - Import bulk data - `CBusinessRecord` - Business record class -- `handelsregister` - Registry automation +- `handelsregister` - German registry automation ## Provider Architecture -The library uses a flexible provider system that makes it easy to add new data sources: +Add custom data providers easily: ```typescript class MyCustomProvider implements IStockProvider { name = 'My Provider'; priority = 50; requiresAuth = true; + rateLimit = { requestsPerMinute: 60 }; - async fetchPrice(request: IStockQuoteRequest): Promise { - // Your implementation - } - - async fetchPrices(request: IStockBatchQuoteRequest): Promise { - // Batch implementation + async fetchData(request: IStockDataRequest): Promise { + // Implement unified data fetching + switch (request.type) { + case 'current': + return this.fetchCurrentPrice(request); + case 'batch': + return this.fetchBatchPrices(request); + case 'historical': + return this.fetchHistoricalPrices(request); + default: + throw new Error(`Unsupported request type`); + } } async isAvailable(): Promise { // Health check + return true; } supportsMarket(market: string): boolean { - // Market validation + return ['US', 'UK', 'DE'].includes(market); } supportsTicker(ticker: string): boolean { - // Ticker validation + return /^[A-Z]{1,5}$/.test(ticker); } } @@ -645,30 +600,53 @@ stockService.register(new MyCustomProvider()); ## Performance -- **Batch fetching**: Get 100+ EOD prices in one API request -- **Smart caching**: Instant repeated queries with configurable TTL -- **Rate limit aware**: Automatic retry logic for API limits -- **Concurrent processing**: Handle 1000+ business records/second +- **Batch Fetching**: Get 100+ prices in one API request +- **Smart Caching**: Data-type aware TTL (historical cached forever, EOD 24h, live 30s) +- **Rate Limit Management**: Automatic retry logic for API limits +- **Concurrent Processing**: Handle 1000+ records/second - **Streaming**: Process GB-sized datasets without memory issues +- **Provider Fallback**: Automatic failover between data sources ## Testing Run the comprehensive test suite: ```bash -npm test +pnpm test ``` -Test stock provider: +Test specific modules: ```bash -npx tstest test/test.marketstack.node.ts --verbose +# Stock price providers +pnpm tstest test/test.marketstack.node.ts --verbose +pnpm tstest test/test.stocks.ts --verbose + +# Fundamental data +pnpm tstest test/test.secedgar.provider.node.ts --verbose +pnpm tstest test/test.fundamentals.service.node.ts --verbose + +# German business data +pnpm tstest test/test.handelsregister.ts --verbose ``` -Test German business data: +## Getting API Keys -```bash -npx tstest test/test.handelsregister.ts --verbose +### Marketstack (EOD Stock Data) + +1. Visit [marketstack.com](https://marketstack.com) +2. Sign up for a free account (100 requests/month) +3. Get your API key from the dashboard +4. Set environment variable: `MARKETSTACK_COM_TOKEN=your_key_here` + +### SEC EDGAR (Fundamental Data) + +**No API key required!** SEC EDGAR is completely free and public. Just provide your company name and email in the User-Agent: + +```typescript +new SecEdgarProvider({ + userAgent: 'YourCompany youremail@example.com' +}); ``` ## License and Legal Information diff --git a/test/test.fundamentals.service.node.ts b/test/test.fundamentals.service.node.ts new file mode 100644 index 0000000..d415d30 --- /dev/null +++ b/test/test.fundamentals.service.node.ts @@ -0,0 +1,287 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import * as opendata from '../ts/index.js'; + +const TEST_USER_AGENT = 'fin.cx test@fin.cx'; + +tap.test('FundamentalsService - Provider Registration', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + await tap.test('should register provider', async () => { + service.register(provider); + + const registered = service.getProvider('SEC EDGAR'); + expect(registered).toBeDefined(); + expect(registered?.name).toEqual('SEC EDGAR'); + }); + + await tap.test('should get all providers', async () => { + const providers = service.getAllProviders(); + expect(providers.length).toBeGreaterThan(0); + expect(providers[0].name).toEqual('SEC EDGAR'); + }); + + await tap.test('should get enabled providers', async () => { + const providers = service.getEnabledProviders(); + expect(providers.length).toBeGreaterThan(0); + }); + + await tap.test('should unregister provider', async () => { + service.unregister('SEC EDGAR'); + + const registered = service.getProvider('SEC EDGAR'); + expect(registered).toBeUndefined(); + + // Re-register for other tests + service.register(provider); + }); +}); + +tap.test('FundamentalsService - Fetch Fundamentals', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should fetch fundamentals for single ticker', async () => { + const fundamentals = await service.getFundamentals('AAPL'); + + expect(fundamentals).toBeDefined(); + expect(fundamentals.ticker).toEqual('AAPL'); + expect(fundamentals.companyName).toEqual('Apple Inc.'); + expect(fundamentals.provider).toEqual('SEC EDGAR'); + expect(fundamentals.earningsPerShareDiluted).toBeGreaterThan(0); + expect(fundamentals.sharesOutstanding).toBeGreaterThan(0); + + console.log('\nšŸ“Š Fetched via Service:'); + console.log(` ${fundamentals.ticker}: ${fundamentals.companyName}`); + console.log(` EPS: $${fundamentals.earningsPerShareDiluted?.toFixed(2)}`); + console.log(` Shares: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`); + }); + + await tap.test('should fetch fundamentals for multiple tickers', async () => { + const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']); + + expect(fundamentalsList).toBeInstanceOf(Array); + expect(fundamentalsList.length).toEqual(2); + + const apple = fundamentalsList.find(f => f.ticker === 'AAPL'); + const msft = fundamentalsList.find(f => f.ticker === 'MSFT'); + + expect(apple).toBeDefined(); + expect(msft).toBeDefined(); + expect(apple!.companyName).toEqual('Apple Inc.'); + expect(msft!.companyName).toContain('Microsoft'); + + console.log('\nšŸ“Š Batch Fetch via Service:'); + fundamentalsList.forEach(f => { + console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`); + }); + }); +}); + +tap.test('FundamentalsService - Caching', async () => { + const service = new opendata.FundamentalsService({ + ttl: 60000, // 60 seconds for testing + maxEntries: 100 + }); + + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should cache fundamentals data', async () => { + // Clear cache first + service.clearCache(); + + let stats = service.getCacheStats(); + expect(stats.size).toEqual(0); + + // First fetch (should hit API) + const start1 = Date.now(); + await service.getFundamentals('AAPL'); + const duration1 = Date.now() - start1; + + stats = service.getCacheStats(); + expect(stats.size).toEqual(1); + + // Second fetch (should hit cache - much faster) + const start2 = Date.now(); + await service.getFundamentals('AAPL'); + const duration2 = Date.now() - start2; + + expect(duration2).toBeLessThan(duration1); + + console.log('\nāš” Cache Performance:'); + console.log(` First fetch: ${duration1}ms`); + console.log(` Cached fetch: ${duration2}ms`); + console.log(` Speedup: ${Math.round(duration1 / duration2)}x`); + }); + + await tap.test('should respect cache TTL', async () => { + // Set very short TTL + service.setCacheTTL(100); // 100ms + + // Fetch and cache + await service.getFundamentals('MSFT'); + + // Wait for TTL to expire + await new Promise(resolve => setTimeout(resolve, 150)); + + // This should fetch again (cache expired) + const stats = service.getCacheStats(); + console.log(`\nā±ļø Cache TTL: ${stats.ttl}ms`); + }); + + await tap.test('should clear cache', async () => { + service.clearCache(); + + const stats = service.getCacheStats(); + expect(stats.size).toEqual(0); + }); +}); + +tap.test('FundamentalsService - Price Enrichment', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should enrich fundamentals with price to calculate market cap', async () => { + const fundamentals = await service.getFundamentals('AAPL'); + + // Simulate current price + const currentPrice = 270.37; + + const enriched = await service.enrichWithPrice(fundamentals, currentPrice); + + expect(enriched.marketCap).toBeDefined(); + expect(enriched.priceToEarnings).toBeDefined(); + expect(enriched.priceToBook).toBeDefined(); + + expect(enriched.marketCap).toBeGreaterThan(0); + expect(enriched.priceToEarnings).toBeGreaterThan(0); + + console.log('\nšŸ’° Enriched with Price ($270.37):'); + console.log(` Market Cap: $${(enriched.marketCap! / 1_000_000_000_000).toFixed(2)}T`); + console.log(` P/E Ratio: ${enriched.priceToEarnings!.toFixed(2)}`); + console.log(` P/B Ratio: ${enriched.priceToBook?.toFixed(2) || 'N/A'}`); + + // Verify calculations + const expectedMarketCap = fundamentals.sharesOutstanding! * currentPrice; + expect(Math.abs(enriched.marketCap! - expectedMarketCap)).toBeLessThan(1); // Allow for rounding + + const expectedPE = currentPrice / fundamentals.earningsPerShareDiluted!; + expect(Math.abs(enriched.priceToEarnings! - expectedPE)).toBeLessThan(0.01); + }); + + await tap.test('should enrich batch fundamentals with prices', async () => { + const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']); + + const priceMap = new Map([ + ['AAPL', 270.37], + ['MSFT', 425.50] + ]); + + const enriched = await service.enrichBatchWithPrices(fundamentalsList, priceMap); + + expect(enriched.length).toEqual(2); + + const apple = enriched.find(f => f.ticker === 'AAPL')!; + const msft = enriched.find(f => f.ticker === 'MSFT')!; + + expect(apple.marketCap).toBeGreaterThan(0); + expect(msft.marketCap).toBeGreaterThan(0); + + console.log('\nšŸ’° Batch Enrichment:'); + console.log(` AAPL: Market Cap $${(apple.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${apple.priceToEarnings!.toFixed(2)}`); + console.log(` MSFT: Market Cap $${(msft.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${msft.priceToEarnings!.toFixed(2)}`); + }); +}); + +tap.test('FundamentalsService - Provider Health', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should check provider health', async () => { + const health = await service.checkProvidersHealth(); + + expect(health.size).toEqual(1); + expect(health.get('SEC EDGAR')).toBe(true); + + console.log('\nšŸ’š Provider Health:'); + health.forEach((isHealthy, name) => { + console.log(` ${name}: ${isHealthy ? 'āœ… Healthy' : 'āŒ Unhealthy'}`); + }); + }); +}); + +tap.test('FundamentalsService - Provider Statistics', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should track provider statistics', async () => { + // Make some requests + await service.getFundamentals('AAPL'); + await service.getFundamentals('MSFT'); + + const stats = service.getProviderStats(); + + expect(stats.size).toEqual(1); + + const secStats = stats.get('SEC EDGAR'); + expect(secStats).toBeDefined(); + expect(secStats!.successCount).toBeGreaterThan(0); + + console.log('\nšŸ“ˆ Provider Stats:'); + console.log(` Success Count: ${secStats!.successCount}`); + console.log(` Error Count: ${secStats!.errorCount}`); + }); +}); + +tap.test('FundamentalsService - Error Handling', async () => { + const service = new opendata.FundamentalsService(); + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + service.register(provider); + + await tap.test('should throw error for invalid ticker', async () => { + try { + await service.getFundamentals('INVALIDTICKER123456'); + throw new Error('Should have thrown error'); + } catch (error) { + expect(error.message).toContain('CIK not found'); + } + }); + + await tap.test('should throw error when no providers available', async () => { + const emptyService = new opendata.FundamentalsService(); + + try { + await emptyService.getFundamentals('AAPL'); + throw new Error('Should have thrown error'); + } catch (error) { + expect(error.message).toContain('No fundamentals providers available'); + } + }); +}); + +export default tap.start(); diff --git a/test/test.secedgar.provider.node.ts b/test/test.secedgar.provider.node.ts new file mode 100644 index 0000000..e30ad48 --- /dev/null +++ b/test/test.secedgar.provider.node.ts @@ -0,0 +1,261 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; +import * as opendata from '../ts/index.js'; + +// Test configuration +const TEST_USER_AGENT = 'fin.cx test@fin.cx'; +const TEST_TICKER = 'AAPL'; // Apple Inc - well-known test case +const RATE_LIMIT_DELAY = 150; // 150ms between requests (< 10 req/sec) + +tap.test('SEC EDGAR Provider - Constructor', async () => { + await tap.test('should create provider with valid User-Agent', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + expect(provider.name).toEqual('SEC EDGAR'); + expect(provider.priority).toEqual(100); + expect(provider.requiresAuth).toBe(false); + expect(provider.rateLimit?.requestsPerMinute).toEqual(600); + }); + + await tap.test('should throw error if User-Agent is missing', async () => { + expect(() => { + new opendata.SecEdgarProvider({ + userAgent: '' + }); + }).toThrow('User-Agent is required'); + }); + + await tap.test('should throw error if User-Agent format is invalid', async () => { + expect(() => { + new opendata.SecEdgarProvider({ + userAgent: 'InvalidFormat' + }); + }).toThrow('Invalid User-Agent format'); + }); + + await tap.test('should accept valid User-Agent with space and email', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: 'MyCompany contact@example.com' + }); + + expect(provider).toBeInstanceOf(opendata.SecEdgarProvider); + }); +}); + +tap.test('SEC EDGAR Provider - Availability', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + await tap.test('should report as available', async () => { + const isAvailable = await provider.isAvailable(); + expect(isAvailable).toBe(true); + }); +}); + +tap.test('SEC EDGAR Provider - Fetch Fundamentals', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT, + timeout: 30000 + }); + + await tap.test('should fetch fundamentals for Apple (AAPL)', async () => { + const fundamentals = await provider.fetchData({ + type: 'fundamentals-current', + ticker: TEST_TICKER + }); + + // Verify structure + expect(fundamentals).toBeDefined(); + expect(fundamentals).not.toBeInstanceOf(Array); + + const data = fundamentals as opendata.IStockFundamentals; + + // Basic fields + expect(data.ticker).toEqual('AAPL'); + expect(data.cik).toBeDefined(); + expect(data.companyName).toEqual('Apple Inc.'); + expect(data.provider).toEqual('SEC EDGAR'); + expect(data.timestamp).toBeInstanceOf(Date); + expect(data.fetchedAt).toBeInstanceOf(Date); + + // Financial metrics (Apple should have all of these) + expect(data.earningsPerShareDiluted).toBeDefined(); + expect(data.earningsPerShareDiluted).toBeGreaterThan(0); + + expect(data.sharesOutstanding).toBeDefined(); + expect(data.sharesOutstanding).toBeGreaterThan(0); + + expect(data.revenue).toBeDefined(); + expect(data.revenue).toBeGreaterThan(0); + + expect(data.netIncome).toBeDefined(); + expect(data.assets).toBeDefined(); + expect(data.assets).toBeGreaterThan(0); + + expect(data.liabilities).toBeDefined(); + expect(data.stockholdersEquity).toBeDefined(); + + // Metadata + expect(data.fiscalYear).toBeDefined(); + + console.log('\nšŸ“Š Sample Apple Fundamentals:'); + console.log(` Company: ${data.companyName} (CIK: ${data.cik})`); + console.log(` EPS (Diluted): $${data.earningsPerShareDiluted?.toFixed(2)}`); + console.log(` Shares Outstanding: ${(data.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`); + console.log(` Revenue: $${(data.revenue! / 1_000_000_000).toFixed(2)}B`); + console.log(` Net Income: $${(data.netIncome! / 1_000_000_000).toFixed(2)}B`); + console.log(` Assets: $${(data.assets! / 1_000_000_000).toFixed(2)}B`); + console.log(` Fiscal Year: ${data.fiscalYear}`); + }); + + await tap.test('should throw error for invalid ticker', async () => { + try { + await provider.fetchData({ + type: 'fundamentals-current', + ticker: 'INVALIDTICKER123456' + }); + throw new Error('Should have thrown error'); + } catch (error) { + expect(error.message).toContain('CIK not found'); + } + }); +}); + +tap.test('SEC EDGAR Provider - Batch Fetch', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT, + timeout: 30000 + }); + + await tap.test('should fetch fundamentals for multiple tickers', async () => { + const result = await provider.fetchData({ + type: 'fundamentals-batch', + tickers: ['AAPL', 'MSFT'] + }); + + expect(result).toBeInstanceOf(Array); + + const fundamentalsList = result as opendata.IStockFundamentals[]; + expect(fundamentalsList.length).toEqual(2); + + // Check Apple + const apple = fundamentalsList.find(f => f.ticker === 'AAPL'); + expect(apple).toBeDefined(); + expect(apple!.companyName).toEqual('Apple Inc.'); + expect(apple!.earningsPerShareDiluted).toBeGreaterThan(0); + + // Check Microsoft + const microsoft = fundamentalsList.find(f => f.ticker === 'MSFT'); + expect(microsoft).toBeDefined(); + expect(microsoft!.companyName).toContain('Microsoft'); + expect(microsoft!.earningsPerShareDiluted).toBeGreaterThan(0); + + console.log('\nšŸ“Š Batch Fetch Results:'); + fundamentalsList.forEach(f => { + console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`); + }); + }); +}); + +tap.test('SEC EDGAR Provider - CIK Caching', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + await tap.test('should cache CIK lookups', async () => { + // Clear cache first + provider.clearCache(); + + let stats = provider.getCacheStats(); + expect(stats.cikCacheSize).toEqual(0); + + // First fetch (should populate cache) + await provider.fetchData({ + type: 'fundamentals-current', + ticker: 'AAPL' + }); + + stats = provider.getCacheStats(); + expect(stats.cikCacheSize).toBeGreaterThan(0); + + console.log(`\nšŸ’¾ CIK Cache: ${stats.cikCacheSize} entries`); + }); + + await tap.test('should clear cache', async () => { + provider.clearCache(); + + const stats = provider.getCacheStats(); + expect(stats.cikCacheSize).toEqual(0); + expect(stats.hasTickerList).toBe(false); + }); +}); + +tap.test('SEC EDGAR Provider - Rate Limiting', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + await tap.test('should handle multiple rapid requests without exceeding rate limit', async () => { + // Make 5 requests in succession + // Rate limiter should ensure we don't exceed 10 req/sec + const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'META']; + const startTime = Date.now(); + + const promises = tickers.map(ticker => + provider.fetchData({ + type: 'fundamentals-current', + ticker + }) + ); + + const results = await Promise.all(promises); + const duration = Date.now() - startTime; + + expect(results.length).toEqual(5); + console.log(`\nā±ļø 5 requests completed in ${duration}ms (avg: ${Math.round(duration / 5)}ms/request)`); + + // Verify all results are valid + results.forEach((result, index) => { + expect(result).toBeDefined(); + const data = result as opendata.IStockFundamentals; + expect(data.ticker).toEqual(tickers[index]); + expect(data.earningsPerShareDiluted).toBeGreaterThan(0); + }); + }); +}); + +tap.test('SEC EDGAR Provider - Market Cap Calculation', async () => { + const provider = new opendata.SecEdgarProvider({ + userAgent: TEST_USER_AGENT + }); + + await tap.test('should provide data needed for market cap calculation', async () => { + const fundamentals = await provider.fetchData({ + type: 'fundamentals-current', + ticker: 'AAPL' + }) as opendata.IStockFundamentals; + + expect(fundamentals.sharesOutstanding).toBeDefined(); + expect(fundamentals.earningsPerShareDiluted).toBeDefined(); + + // Simulate current price (in real usage, this comes from price provider) + const simulatedPrice = 270.37; + + // Calculate market cap + const marketCap = fundamentals.sharesOutstanding! * simulatedPrice; + const pe = simulatedPrice / fundamentals.earningsPerShareDiluted!; + + console.log('\nšŸ’° Calculated Metrics (with simulated price $270.37):'); + console.log(` Shares Outstanding: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`); + console.log(` Market Cap: $${(marketCap / 1_000_000_000_000).toFixed(2)}T`); + console.log(` EPS: $${fundamentals.earningsPerShareDiluted!.toFixed(2)}`); + console.log(` P/E Ratio: ${pe.toFixed(2)}`); + + expect(marketCap).toBeGreaterThan(0); + expect(pe).toBeGreaterThan(0); + }); +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index b72a1f5..b4ff112 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@fin.cx/opendata', - version: '3.0.0', + version: '3.1.0', description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.' } diff --git a/ts/stocks/classes.fundamentalsservice.ts b/ts/stocks/classes.fundamentalsservice.ts new file mode 100644 index 0000000..5e15aff --- /dev/null +++ b/ts/stocks/classes.fundamentalsservice.ts @@ -0,0 +1,404 @@ +import * as plugins from '../plugins.js'; +import type { + IFundamentalsProvider, + IFundamentalsProviderConfig, + IFundamentalsProviderRegistry, + IStockFundamentals, + IFundamentalsRequest +} from './interfaces/fundamentals.js'; + +interface IProviderEntry { + provider: IFundamentalsProvider; + config: IFundamentalsProviderConfig; + lastError?: Error; + lastErrorTime?: Date; + successCount: number; + errorCount: number; +} + +interface ICacheEntry { + fundamentals: IStockFundamentals | IStockFundamentals[]; + timestamp: Date; + ttl: number; +} + +/** + * Service for managing fundamental data providers and caching + * Parallel to StockPriceService but for fundamental data instead of prices + */ +export class FundamentalsService implements IFundamentalsProviderRegistry { + private providers = new Map(); + private cache = new Map(); + private logger = console; + + private cacheConfig = { + ttl: 90 * 24 * 60 * 60 * 1000, // 90 days default (fundamentals change quarterly) + maxEntries: 10000 + }; + + constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) { + if (cacheConfig) { + this.cacheConfig = { ...this.cacheConfig, ...cacheConfig }; + } + } + + /** + * Register a fundamentals provider + */ + public register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void { + const defaultConfig: IFundamentalsProviderConfig = { + enabled: true, + priority: provider.priority, + timeout: 30000, // Longer timeout for fundamental data + retryAttempts: 2, + retryDelay: 1000, + cacheTTL: this.cacheConfig.ttl + }; + + const mergedConfig = { ...defaultConfig, ...config }; + + this.providers.set(provider.name, { + provider, + config: mergedConfig, + successCount: 0, + errorCount: 0 + }); + + console.log(`Registered fundamentals provider: ${provider.name}`); + } + + /** + * Unregister a provider + */ + public unregister(providerName: string): void { + this.providers.delete(providerName); + console.log(`Unregistered fundamentals provider: ${providerName}`); + } + + /** + * Get a specific provider by name + */ + public getProvider(name: string): IFundamentalsProvider | undefined { + return this.providers.get(name)?.provider; + } + + /** + * Get all registered providers + */ + public getAllProviders(): IFundamentalsProvider[] { + return Array.from(this.providers.values()).map(entry => entry.provider); + } + + /** + * Get enabled providers sorted by priority + */ + public getEnabledProviders(): IFundamentalsProvider[] { + return Array.from(this.providers.values()) + .filter(entry => entry.config.enabled) + .sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0)) + .map(entry => entry.provider); + } + + /** + * Get fundamental data for a single ticker + */ + public async getFundamentals(ticker: string): Promise { + const result = await this.getData({ + type: 'fundamentals-current', + ticker + }); + return result as IStockFundamentals; + } + + /** + * Get fundamental data for multiple tickers + */ + public async getBatchFundamentals(tickers: string[]): Promise { + const result = await this.getData({ + type: 'fundamentals-batch', + tickers + }); + return result as IStockFundamentals[]; + } + + /** + * Unified data fetching method + */ + public async getData( + request: IFundamentalsRequest + ): Promise { + const cacheKey = this.getCacheKey(request); + const cached = this.getFromCache(cacheKey); + + if (cached) { + console.log(`Cache hit for ${this.getRequestDescription(request)}`); + return cached; + } + + const providers = this.getEnabledProviders(); + if (providers.length === 0) { + throw new Error('No fundamentals providers available'); + } + + let lastError: Error | undefined; + + for (const provider of providers) { + const entry = this.providers.get(provider.name)!; + + try { + const result = await this.fetchWithRetry( + () => provider.fetchData(request), + entry.config + ); + + entry.successCount++; + + // Use provider-specific cache TTL or default + const ttl = entry.config.cacheTTL || this.cacheConfig.ttl; + this.addToCache(cacheKey, result, ttl); + + console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`); + return result; + } catch (error) { + entry.errorCount++; + entry.lastError = error as Error; + entry.lastErrorTime = new Date(); + lastError = error as Error; + + console.warn( + `Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}` + ); + } + } + + throw new Error( + `Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}` + ); + } + + /** + * Enrich fundamentals with calculated metrics using current price + */ + public async enrichWithPrice( + fundamentals: IStockFundamentals, + price: number + ): Promise { + const enriched = { ...fundamentals }; + + // Calculate market cap: price Ɨ shares outstanding + if (fundamentals.sharesOutstanding) { + enriched.marketCap = price * fundamentals.sharesOutstanding; + } + + // Calculate P/E ratio: price / EPS + if (fundamentals.earningsPerShareDiluted && fundamentals.earningsPerShareDiluted > 0) { + enriched.priceToEarnings = price / fundamentals.earningsPerShareDiluted; + } + + // Calculate price-to-book: market cap / stockholders equity + if (enriched.marketCap && fundamentals.stockholdersEquity && fundamentals.stockholdersEquity > 0) { + enriched.priceToBook = enriched.marketCap / fundamentals.stockholdersEquity; + } + + return enriched; + } + + /** + * Enrich batch fundamentals with prices + */ + public async enrichBatchWithPrices( + fundamentalsList: IStockFundamentals[], + priceMap: Map + ): Promise { + return Promise.all( + fundamentalsList.map(fundamentals => { + const price = priceMap.get(fundamentals.ticker); + if (price) { + return this.enrichWithPrice(fundamentals, price); + } + return Promise.resolve(fundamentals); + }) + ); + } + + /** + * Check health of all providers + */ + public async checkProvidersHealth(): Promise> { + const health = new Map(); + + for (const [name, entry] of this.providers) { + if (!entry.config.enabled) { + health.set(name, false); + continue; + } + + try { + const isAvailable = await entry.provider.isAvailable(); + health.set(name, isAvailable); + } catch (error) { + health.set(name, false); + console.error(`Health check failed for ${name}:`, error); + } + } + + return health; + } + + /** + * Get provider statistics + */ + public getProviderStats(): Map< + string, + { + successCount: number; + errorCount: number; + lastError?: string; + lastErrorTime?: Date; + } + > { + const stats = new Map(); + + for (const [name, entry] of this.providers) { + stats.set(name, { + successCount: entry.successCount, + errorCount: entry.errorCount, + lastError: entry.lastError?.message, + lastErrorTime: entry.lastErrorTime + }); + } + + return stats; + } + + /** + * Clear all cached data + */ + public clearCache(): void { + this.cache.clear(); + console.log('Fundamentals cache cleared'); + } + + /** + * Set cache TTL + */ + public setCacheTTL(ttl: number): void { + this.cacheConfig.ttl = ttl; + console.log(`Fundamentals cache TTL set to ${ttl}ms`); + } + + /** + * Get cache statistics + */ + public getCacheStats(): { + size: number; + maxEntries: number; + ttl: number; + } { + return { + size: this.cache.size, + maxEntries: this.cacheConfig.maxEntries, + ttl: this.cacheConfig.ttl + }; + } + + /** + * Fetch with retry logic + */ + private async fetchWithRetry( + fetchFn: () => Promise, + config: IFundamentalsProviderConfig + ): Promise { + const maxAttempts = config.retryAttempts || 1; + let lastError: Error | undefined; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await fetchFn(); + } catch (error) { + lastError = error as Error; + + if (attempt < maxAttempts) { + const delay = (config.retryDelay || 1000) * attempt; + console.log(`Retry attempt ${attempt} after ${delay}ms`); + await plugins.smartdelay.delayFor(delay); + } + } + } + + throw lastError || new Error('Unknown error during fetch'); + } + + /** + * Generate cache key for request + */ + private getCacheKey(request: IFundamentalsRequest): string { + switch (request.type) { + case 'fundamentals-current': + return `fundamentals:${request.ticker}`; + case 'fundamentals-batch': + const tickers = request.tickers.sort().join(','); + return `fundamentals-batch:${tickers}`; + default: + return `unknown:${JSON.stringify(request)}`; + } + } + + /** + * Get from cache if not expired + */ + private getFromCache(key: string): IStockFundamentals | IStockFundamentals[] | null { + const entry = this.cache.get(key); + + if (!entry) { + return null; + } + + // Check if cache entry has expired + const age = Date.now() - entry.timestamp.getTime(); + if (entry.ttl !== Infinity && age > entry.ttl) { + this.cache.delete(key); + return null; + } + + return entry.fundamentals; + } + + /** + * Add to cache with TTL + */ + private addToCache( + key: string, + fundamentals: IStockFundamentals | IStockFundamentals[], + ttl?: number + ): void { + // Enforce max entries limit + if (this.cache.size >= this.cacheConfig.maxEntries) { + // Remove oldest entry + const oldestKey = this.cache.keys().next().value; + if (oldestKey) { + this.cache.delete(oldestKey); + } + } + + this.cache.set(key, { + fundamentals, + timestamp: new Date(), + ttl: ttl || this.cacheConfig.ttl + }); + } + + /** + * Get human-readable request description + */ + private getRequestDescription(request: IFundamentalsRequest): string { + switch (request.type) { + case 'fundamentals-current': + return `fundamentals for ${request.ticker}`; + case 'fundamentals-batch': + return `fundamentals for ${request.tickers.length} tickers`; + default: + return 'fundamentals data'; + } + } +} diff --git a/ts/stocks/index.ts b/ts/stocks/index.ts index 94da2fc..15abd7f 100644 --- a/ts/stocks/index.ts +++ b/ts/stocks/index.ts @@ -1,10 +1,13 @@ // Export all interfaces export * from './interfaces/stockprice.js'; export * from './interfaces/provider.js'; +export * from './interfaces/fundamentals.js'; -// Export main service +// Export main services export * from './classes.stockservice.js'; +export * from './classes.fundamentalsservice.js'; // Export providers export * from './providers/provider.yahoo.js'; -export * from './providers/provider.marketstack.js'; \ No newline at end of file +export * from './providers/provider.marketstack.js'; +export * from './providers/provider.secedgar.js'; \ No newline at end of file diff --git a/ts/stocks/interfaces/fundamentals.ts b/ts/stocks/interfaces/fundamentals.ts new file mode 100644 index 0000000..3075afc --- /dev/null +++ b/ts/stocks/interfaces/fundamentals.ts @@ -0,0 +1,102 @@ +/** + * Interfaces for stock fundamental data (financials from SEC filings) + * Separate from stock price data (OHLCV) to maintain clean architecture + */ + +// Request types for fundamental data +export interface IFundamentalsCurrentRequest { + type: 'fundamentals-current'; + ticker: string; +} + +export interface IFundamentalsBatchRequest { + type: 'fundamentals-batch'; + tickers: string[]; +} + +export type IFundamentalsRequest = + | IFundamentalsCurrentRequest + | IFundamentalsBatchRequest; + +/** + * Stock fundamental data from SEC filings (10-K, 10-Q) + * Contains financial metrics like EPS, Revenue, Assets, etc. + */ +export interface IStockFundamentals { + ticker: string; + cik: string; + companyName: string; + provider: string; + timestamp: Date; + fetchedAt: Date; + + // Per-share metrics + earningsPerShareBasic?: number; + earningsPerShareDiluted?: number; + sharesOutstanding?: number; + weightedAverageSharesOutstanding?: number; + + // Income statement (annual USD) + revenue?: number; + netIncome?: number; + operatingIncome?: number; + grossProfit?: number; + costOfRevenue?: number; + + // Balance sheet (annual USD) + assets?: number; + liabilities?: number; + stockholdersEquity?: number; + cash?: number; + propertyPlantEquipment?: number; + + // Calculated metrics (requires current price) + marketCap?: number; // price Ɨ sharesOutstanding + priceToEarnings?: number; // price / EPS + priceToBook?: number; // marketCap / stockholdersEquity + + // Metadata + fiscalYear?: string; + fiscalQuarter?: string; + filingDate?: Date; + form?: '10-K' | '10-Q' | string; +} + +/** + * Provider interface for fetching fundamental data + * Parallel to IStockProvider but for fundamentals instead of prices + */ +export interface IFundamentalsProvider { + name: string; + priority: number; + fetchData(request: IFundamentalsRequest): Promise; + isAvailable(): Promise; + readonly requiresAuth: boolean; + readonly rateLimit?: { + requestsPerMinute: number; + requestsPerDay?: number; + }; +} + +/** + * Configuration for fundamentals providers + */ +export interface IFundamentalsProviderConfig { + enabled: boolean; + priority?: number; + timeout?: number; + retryAttempts?: number; + retryDelay?: number; + cacheTTL?: number; // Custom cache TTL for this provider +} + +/** + * Registry for managing fundamental data providers + */ +export interface IFundamentalsProviderRegistry { + register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void; + unregister(providerName: string): void; + getProvider(name: string): IFundamentalsProvider | undefined; + getAllProviders(): IFundamentalsProvider[]; + getEnabledProviders(): IFundamentalsProvider[]; +} diff --git a/ts/stocks/providers/provider.secedgar.ts b/ts/stocks/providers/provider.secedgar.ts new file mode 100644 index 0000000..8bcfbb1 --- /dev/null +++ b/ts/stocks/providers/provider.secedgar.ts @@ -0,0 +1,429 @@ +import * as plugins from '../../plugins.js'; +import type { + IFundamentalsProvider, + IStockFundamentals, + IFundamentalsRequest, + IFundamentalsCurrentRequest, + IFundamentalsBatchRequest +} from '../interfaces/fundamentals.js'; + +/** + * Configuration for SEC EDGAR provider + */ +export interface ISecEdgarConfig { + userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx") + cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change) + fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings) + timeout?: number; // Request timeout in ms +} + +/** + * Rate limiter for SEC EDGAR API + * SEC requires: 10 requests per second maximum + */ +class RateLimiter { + private requestTimes: number[] = []; + private readonly maxRequestsPerSecond = 10; + + public async waitForSlot(): Promise { + const now = Date.now(); + const oneSecondAgo = now - 1000; + + // Remove requests older than 1 second + this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo); + + // If we've hit the limit, wait + if (this.requestTimes.length >= this.maxRequestsPerSecond) { + const oldestRequest = this.requestTimes[0]; + const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer + await plugins.smartdelay.delayFor(waitTime); + return this.waitForSlot(); // Recursively check again + } + + // Record this request + this.requestTimes.push(now); + } +} + +/** + * SEC EDGAR Fundamental Data Provider + * + * Features: + * - Free public access (no API key required) + * - All US public companies + * - Financial data from 10-K/10-Q filings + * - US GAAP standardized metrics + * - Historical data back to ~2009 + * - < 1 minute filing delay + * + * Documentation: https://www.sec.gov/edgar/sec-api-documentation + * + * Rate Limits: + * - 10 requests per second (enforced by SEC) + * - Requires User-Agent header in format: "Company Name Email" + * + * Data Sources: + * - Company Facts API: /api/xbrl/companyfacts/CIK##########.json + * - Ticker Lookup: /files/company_tickers.json + */ +export class SecEdgarProvider implements IFundamentalsProvider { + public name = 'SEC EDGAR'; + public priority = 100; // High priority - free, authoritative, comprehensive + public readonly requiresAuth = false; // No API key needed! + public readonly rateLimit = { + requestsPerMinute: 600, // 10 requests/second = 600/minute + requestsPerDay: undefined // No daily limit + }; + + private logger = console; + private baseUrl = 'https://data.sec.gov/api/xbrl'; + private tickersUrl = 'https://www.sec.gov/files/company_tickers.json'; + private userAgent: string; + private config: Required; + + // Caching + private cikCache = new Map(); + private tickerListCache: { data: any; timestamp: Date } | null = null; + + // Rate limiting + private rateLimiter = new RateLimiter(); + + constructor(config: ISecEdgarConfig) { + // Validate User-Agent + if (!config.userAgent) { + throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")'); + } + + // Validate User-Agent format (must contain space and @ symbol) + if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) { + throw new Error( + 'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")' + ); + } + + this.userAgent = config.userAgent; + this.config = { + userAgent: config.userAgent, + cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days + fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days + timeout: config.timeout || 30000 + }; + } + + /** + * Unified data fetching method + */ + public async fetchData( + request: IFundamentalsRequest + ): Promise { + switch (request.type) { + case 'fundamentals-current': + return this.fetchFundamentals(request); + case 'fundamentals-batch': + return this.fetchBatchFundamentals(request); + default: + throw new Error(`Unsupported request type: ${(request as any).type}`); + } + } + + /** + * Fetch fundamental data for a single ticker + */ + private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise { + try { + // 1. Get CIK for ticker (with caching) + const cik = await this.getCIK(request.ticker); + + // 2. Fetch company facts from SEC (with rate limiting) + const companyFacts = await this.fetchCompanyFacts(cik); + + // 3. Parse facts into structured fundamental data + return this.parseCompanyFacts(request.ticker, cik, companyFacts); + } catch (error) { + this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error); + throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`); + } + } + + /** + * Fetch fundamental data for multiple tickers + */ + private async fetchBatchFundamentals( + request: IFundamentalsBatchRequest + ): Promise { + const results: IStockFundamentals[] = []; + const errors: string[] = []; + + for (const ticker of request.tickers) { + try { + const fundamentals = await this.fetchFundamentals({ + type: 'fundamentals-current', + ticker + }); + results.push(fundamentals); + } catch (error) { + this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error); + errors.push(`${ticker}: ${error.message}`); + // Continue with other tickers + } + } + + if (results.length === 0) { + throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`); + } + + return results; + } + + /** + * Get CIK (Central Index Key) for a ticker symbol + * Uses SEC's public ticker-to-CIK mapping file + */ + private async getCIK(ticker: string): Promise { + const tickerUpper = ticker.toUpperCase(); + + // Check cache first + const cached = this.cikCache.get(tickerUpper); + if (cached) { + const age = Date.now() - cached.timestamp.getTime(); + if (age < this.config.cikCacheTTL) { + return cached.cik; + } + // Cache expired, remove it + this.cikCache.delete(tickerUpper); + } + + // Fetch ticker list (with caching at list level) + const tickers = await this.fetchTickerList(); + + // Find ticker in list (case-insensitive) + const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper); + + if (!entry) { + throw new Error(`CIK not found for ticker ${ticker}`); + } + + const cik = String((entry as any).cik_str); + + // Cache the result + this.cikCache.set(tickerUpper, { + cik, + timestamp: new Date() + }); + + return cik; + } + + /** + * Fetch the SEC ticker-to-CIK mapping list + * Cached for 24 hours (list updates daily) + */ + private async fetchTickerList(): Promise { + // Check cache + if (this.tickerListCache) { + const age = Date.now() - this.tickerListCache.timestamp.getTime(); + if (age < 24 * 60 * 60 * 1000) { // 24 hours + return this.tickerListCache.data; + } + } + + // Wait for rate limit slot + await this.rateLimiter.waitForSlot(); + + // Fetch from SEC + const response = await plugins.smartrequest.SmartRequest.create() + .url(this.tickersUrl) + .headers({ + 'User-Agent': this.userAgent, + 'Accept': 'application/json' + }) + .timeout(this.config.timeout) + .get(); + + const data = await response.json(); + + // Cache the list + this.tickerListCache = { + data, + timestamp: new Date() + }; + + return data; + } + + /** + * Fetch company facts from SEC EDGAR + */ + private async fetchCompanyFacts(cik: string): Promise { + // Pad CIK to 10 digits + const paddedCIK = cik.padStart(10, '0'); + const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`; + + // Wait for rate limit slot + await this.rateLimiter.waitForSlot(); + + // Fetch from SEC + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .headers({ + 'User-Agent': this.userAgent, + 'Accept': 'application/json', + 'Accept-Encoding': 'gzip, deflate', + 'Host': 'data.sec.gov' + }) + .timeout(this.config.timeout) + .get(); + + const data = await response.json(); + + // Validate response + if (!data || !data.facts) { + throw new Error('Invalid response from SEC EDGAR API'); + } + + return data; + } + + /** + * Parse SEC company facts into structured fundamental data + */ + private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals { + const usGaap = data.facts?.['us-gaap']; + + if (!usGaap) { + throw new Error('No US GAAP data available'); + } + + // Extract latest values for key metrics + const fundamentals: IStockFundamentals = { + ticker: ticker.toUpperCase(), + cik: cik, + companyName: data.entityName, + provider: this.name, + timestamp: new Date(), + fetchedAt: new Date(), + + // Per-share metrics + earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'), + earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'), + sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'), + weightedAverageSharesOutstanding: this.getLatestValue( + usGaap, + 'WeightedAverageNumberOfSharesOutstandingBasic' + ), + + // Income statement + revenue: this.getLatestValue(usGaap, 'Revenues') || + this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'), + netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'), + operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'), + grossProfit: this.getLatestValue(usGaap, 'GrossProfit'), + costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'), + + // Balance sheet + assets: this.getLatestValue(usGaap, 'Assets'), + liabilities: this.getLatestValue(usGaap, 'Liabilities'), + stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'), + cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'), + propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'), + + // Metadata (from latest available data point) + fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy, + fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp, + filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed + ? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed) + : undefined, + form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form + }; + + return fundamentals; + } + + /** + * Get the latest value for a US GAAP metric + */ + private getLatestValue(usGaap: any, metricName: string): number | undefined { + const metric = usGaap[metricName]; + if (!metric?.units) { + return undefined; + } + + // Get the first unit type (USD, shares, etc.) + const unitType = Object.keys(metric.units)[0]; + const values = metric.units[unitType]; + + if (!values || !Array.isArray(values) || values.length === 0) { + return undefined; + } + + // Get the latest value (last in array) + const latest = values[values.length - 1]; + return latest?.val; + } + + /** + * Get metadata from the latest data point + */ + private getLatestMetadata(usGaap: any, metricName: string): any | undefined { + const metric = usGaap[metricName]; + if (!metric?.units) { + return undefined; + } + + const unitType = Object.keys(metric.units)[0]; + const values = metric.units[unitType]; + + if (!values || !Array.isArray(values) || values.length === 0) { + return undefined; + } + + return values[values.length - 1]; + } + + /** + * Check if SEC EDGAR API is available + */ + public async isAvailable(): Promise { + try { + // Test with Apple's well-known CIK + const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`; + + const response = await plugins.smartrequest.SmartRequest.create() + .url(url) + .headers({ + 'User-Agent': this.userAgent, + 'Accept': 'application/json' + }) + .timeout(5000) + .get(); + + const data = await response.json(); + return data && data.facts !== undefined; + } catch (error) { + this.logger.warn('SEC EDGAR provider is not available:', error); + return false; + } + } + + /** + * Get cache statistics + */ + public getCacheStats(): { + cikCacheSize: number; + hasTickerList: boolean; + } { + return { + cikCacheSize: this.cikCache.size, + hasTickerList: this.tickerListCache !== null + }; + } + + /** + * Clear all caches + */ + public clearCache(): void { + this.cikCache.clear(); + this.tickerListCache = null; + this.logger.log('SEC EDGAR cache cleared'); + } +}