feat(fundamentals): Add FundamentalsService and SEC EDGAR provider with caching, rate-limiting, tests, and docs updates
This commit is contained in:
11
changelog.md
11
changelog.md
@@ -1,5 +1,16 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-11-01 - 3.1.0 - feat(fundamentals)
|
||||
Add FundamentalsService and SEC EDGAR provider with caching, rate-limiting, tests, and docs updates
|
||||
|
||||
- Introduce FundamentalsService to manage fundamentals providers, caching, retry logic and provider statistics
|
||||
- Add SecEdgarProvider to fetch SEC EDGAR company facts (CIK lookup, company facts parsing) with rate limiting and local caches
|
||||
- Expose fundamentals interfaces and services from ts/stocks (exports updated)
|
||||
- Add comprehensive tests for FundamentalsService and SecEdgarProvider (new test files)
|
||||
- Update README with new Fundamentals module documentation, usage examples, and configuration guidance
|
||||
- Implement caching and TTL handling for fundamentals data and provider-specific cache TTL support
|
||||
- Add .claude/settings.local.json (local permissions) and various test improvements
|
||||
|
||||
## 2025-10-31 - 3.0.0 - BREAKING CHANGE(stocks)
|
||||
Unify stock provider API to discriminated IStockDataRequest and add company name/fullname enrichment
|
||||
|
||||
|
||||
287
test/test.fundamentals.service.node.ts
Normal file
287
test/test.fundamentals.service.node.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js';
|
||||
|
||||
const TEST_USER_AGENT = 'fin.cx test@fin.cx';
|
||||
|
||||
tap.test('FundamentalsService - Provider Registration', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
await tap.test('should register provider', async () => {
|
||||
service.register(provider);
|
||||
|
||||
const registered = service.getProvider('SEC EDGAR');
|
||||
expect(registered).toBeDefined();
|
||||
expect(registered?.name).toEqual('SEC EDGAR');
|
||||
});
|
||||
|
||||
await tap.test('should get all providers', async () => {
|
||||
const providers = service.getAllProviders();
|
||||
expect(providers.length).toBeGreaterThan(0);
|
||||
expect(providers[0].name).toEqual('SEC EDGAR');
|
||||
});
|
||||
|
||||
await tap.test('should get enabled providers', async () => {
|
||||
const providers = service.getEnabledProviders();
|
||||
expect(providers.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
await tap.test('should unregister provider', async () => {
|
||||
service.unregister('SEC EDGAR');
|
||||
|
||||
const registered = service.getProvider('SEC EDGAR');
|
||||
expect(registered).toBeUndefined();
|
||||
|
||||
// Re-register for other tests
|
||||
service.register(provider);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Fetch Fundamentals', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should fetch fundamentals for single ticker', async () => {
|
||||
const fundamentals = await service.getFundamentals('AAPL');
|
||||
|
||||
expect(fundamentals).toBeDefined();
|
||||
expect(fundamentals.ticker).toEqual('AAPL');
|
||||
expect(fundamentals.companyName).toEqual('Apple Inc.');
|
||||
expect(fundamentals.provider).toEqual('SEC EDGAR');
|
||||
expect(fundamentals.earningsPerShareDiluted).toBeGreaterThan(0);
|
||||
expect(fundamentals.sharesOutstanding).toBeGreaterThan(0);
|
||||
|
||||
console.log('\n📊 Fetched via Service:');
|
||||
console.log(` ${fundamentals.ticker}: ${fundamentals.companyName}`);
|
||||
console.log(` EPS: $${fundamentals.earningsPerShareDiluted?.toFixed(2)}`);
|
||||
console.log(` Shares: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
|
||||
});
|
||||
|
||||
await tap.test('should fetch fundamentals for multiple tickers', async () => {
|
||||
const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']);
|
||||
|
||||
expect(fundamentalsList).toBeInstanceOf(Array);
|
||||
expect(fundamentalsList.length).toEqual(2);
|
||||
|
||||
const apple = fundamentalsList.find(f => f.ticker === 'AAPL');
|
||||
const msft = fundamentalsList.find(f => f.ticker === 'MSFT');
|
||||
|
||||
expect(apple).toBeDefined();
|
||||
expect(msft).toBeDefined();
|
||||
expect(apple!.companyName).toEqual('Apple Inc.');
|
||||
expect(msft!.companyName).toContain('Microsoft');
|
||||
|
||||
console.log('\n📊 Batch Fetch via Service:');
|
||||
fundamentalsList.forEach(f => {
|
||||
console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Caching', async () => {
|
||||
const service = new opendata.FundamentalsService({
|
||||
ttl: 60000, // 60 seconds for testing
|
||||
maxEntries: 100
|
||||
});
|
||||
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should cache fundamentals data', async () => {
|
||||
// Clear cache first
|
||||
service.clearCache();
|
||||
|
||||
let stats = service.getCacheStats();
|
||||
expect(stats.size).toEqual(0);
|
||||
|
||||
// First fetch (should hit API)
|
||||
const start1 = Date.now();
|
||||
await service.getFundamentals('AAPL');
|
||||
const duration1 = Date.now() - start1;
|
||||
|
||||
stats = service.getCacheStats();
|
||||
expect(stats.size).toEqual(1);
|
||||
|
||||
// Second fetch (should hit cache - much faster)
|
||||
const start2 = Date.now();
|
||||
await service.getFundamentals('AAPL');
|
||||
const duration2 = Date.now() - start2;
|
||||
|
||||
expect(duration2).toBeLessThan(duration1);
|
||||
|
||||
console.log('\n⚡ Cache Performance:');
|
||||
console.log(` First fetch: ${duration1}ms`);
|
||||
console.log(` Cached fetch: ${duration2}ms`);
|
||||
console.log(` Speedup: ${Math.round(duration1 / duration2)}x`);
|
||||
});
|
||||
|
||||
await tap.test('should respect cache TTL', async () => {
|
||||
// Set very short TTL
|
||||
service.setCacheTTL(100); // 100ms
|
||||
|
||||
// Fetch and cache
|
||||
await service.getFundamentals('MSFT');
|
||||
|
||||
// Wait for TTL to expire
|
||||
await new Promise(resolve => setTimeout(resolve, 150));
|
||||
|
||||
// This should fetch again (cache expired)
|
||||
const stats = service.getCacheStats();
|
||||
console.log(`\n⏱️ Cache TTL: ${stats.ttl}ms`);
|
||||
});
|
||||
|
||||
await tap.test('should clear cache', async () => {
|
||||
service.clearCache();
|
||||
|
||||
const stats = service.getCacheStats();
|
||||
expect(stats.size).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Price Enrichment', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should enrich fundamentals with price to calculate market cap', async () => {
|
||||
const fundamentals = await service.getFundamentals('AAPL');
|
||||
|
||||
// Simulate current price
|
||||
const currentPrice = 270.37;
|
||||
|
||||
const enriched = await service.enrichWithPrice(fundamentals, currentPrice);
|
||||
|
||||
expect(enriched.marketCap).toBeDefined();
|
||||
expect(enriched.priceToEarnings).toBeDefined();
|
||||
expect(enriched.priceToBook).toBeDefined();
|
||||
|
||||
expect(enriched.marketCap).toBeGreaterThan(0);
|
||||
expect(enriched.priceToEarnings).toBeGreaterThan(0);
|
||||
|
||||
console.log('\n💰 Enriched with Price ($270.37):');
|
||||
console.log(` Market Cap: $${(enriched.marketCap! / 1_000_000_000_000).toFixed(2)}T`);
|
||||
console.log(` P/E Ratio: ${enriched.priceToEarnings!.toFixed(2)}`);
|
||||
console.log(` P/B Ratio: ${enriched.priceToBook?.toFixed(2) || 'N/A'}`);
|
||||
|
||||
// Verify calculations
|
||||
const expectedMarketCap = fundamentals.sharesOutstanding! * currentPrice;
|
||||
expect(Math.abs(enriched.marketCap! - expectedMarketCap)).toBeLessThan(1); // Allow for rounding
|
||||
|
||||
const expectedPE = currentPrice / fundamentals.earningsPerShareDiluted!;
|
||||
expect(Math.abs(enriched.priceToEarnings! - expectedPE)).toBeLessThan(0.01);
|
||||
});
|
||||
|
||||
await tap.test('should enrich batch fundamentals with prices', async () => {
|
||||
const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']);
|
||||
|
||||
const priceMap = new Map<string, number>([
|
||||
['AAPL', 270.37],
|
||||
['MSFT', 425.50]
|
||||
]);
|
||||
|
||||
const enriched = await service.enrichBatchWithPrices(fundamentalsList, priceMap);
|
||||
|
||||
expect(enriched.length).toEqual(2);
|
||||
|
||||
const apple = enriched.find(f => f.ticker === 'AAPL')!;
|
||||
const msft = enriched.find(f => f.ticker === 'MSFT')!;
|
||||
|
||||
expect(apple.marketCap).toBeGreaterThan(0);
|
||||
expect(msft.marketCap).toBeGreaterThan(0);
|
||||
|
||||
console.log('\n💰 Batch Enrichment:');
|
||||
console.log(` AAPL: Market Cap $${(apple.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${apple.priceToEarnings!.toFixed(2)}`);
|
||||
console.log(` MSFT: Market Cap $${(msft.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${msft.priceToEarnings!.toFixed(2)}`);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Provider Health', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should check provider health', async () => {
|
||||
const health = await service.checkProvidersHealth();
|
||||
|
||||
expect(health.size).toEqual(1);
|
||||
expect(health.get('SEC EDGAR')).toBe(true);
|
||||
|
||||
console.log('\n💚 Provider Health:');
|
||||
health.forEach((isHealthy, name) => {
|
||||
console.log(` ${name}: ${isHealthy ? '✅ Healthy' : '❌ Unhealthy'}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Provider Statistics', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should track provider statistics', async () => {
|
||||
// Make some requests
|
||||
await service.getFundamentals('AAPL');
|
||||
await service.getFundamentals('MSFT');
|
||||
|
||||
const stats = service.getProviderStats();
|
||||
|
||||
expect(stats.size).toEqual(1);
|
||||
|
||||
const secStats = stats.get('SEC EDGAR');
|
||||
expect(secStats).toBeDefined();
|
||||
expect(secStats!.successCount).toBeGreaterThan(0);
|
||||
|
||||
console.log('\n📈 Provider Stats:');
|
||||
console.log(` Success Count: ${secStats!.successCount}`);
|
||||
console.log(` Error Count: ${secStats!.errorCount}`);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('FundamentalsService - Error Handling', async () => {
|
||||
const service = new opendata.FundamentalsService();
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
service.register(provider);
|
||||
|
||||
await tap.test('should throw error for invalid ticker', async () => {
|
||||
try {
|
||||
await service.getFundamentals('INVALIDTICKER123456');
|
||||
throw new Error('Should have thrown error');
|
||||
} catch (error) {
|
||||
expect(error.message).toContain('CIK not found');
|
||||
}
|
||||
});
|
||||
|
||||
await tap.test('should throw error when no providers available', async () => {
|
||||
const emptyService = new opendata.FundamentalsService();
|
||||
|
||||
try {
|
||||
await emptyService.getFundamentals('AAPL');
|
||||
throw new Error('Should have thrown error');
|
||||
} catch (error) {
|
||||
expect(error.message).toContain('No fundamentals providers available');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
261
test/test.secedgar.provider.node.ts
Normal file
261
test/test.secedgar.provider.node.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js';
|
||||
|
||||
// Test configuration
|
||||
const TEST_USER_AGENT = 'fin.cx test@fin.cx';
|
||||
const TEST_TICKER = 'AAPL'; // Apple Inc - well-known test case
|
||||
const RATE_LIMIT_DELAY = 150; // 150ms between requests (< 10 req/sec)
|
||||
|
||||
tap.test('SEC EDGAR Provider - Constructor', async () => {
|
||||
await tap.test('should create provider with valid User-Agent', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
expect(provider.name).toEqual('SEC EDGAR');
|
||||
expect(provider.priority).toEqual(100);
|
||||
expect(provider.requiresAuth).toBe(false);
|
||||
expect(provider.rateLimit?.requestsPerMinute).toEqual(600);
|
||||
});
|
||||
|
||||
await tap.test('should throw error if User-Agent is missing', async () => {
|
||||
expect(() => {
|
||||
new opendata.SecEdgarProvider({
|
||||
userAgent: ''
|
||||
});
|
||||
}).toThrow('User-Agent is required');
|
||||
});
|
||||
|
||||
await tap.test('should throw error if User-Agent format is invalid', async () => {
|
||||
expect(() => {
|
||||
new opendata.SecEdgarProvider({
|
||||
userAgent: 'InvalidFormat'
|
||||
});
|
||||
}).toThrow('Invalid User-Agent format');
|
||||
});
|
||||
|
||||
await tap.test('should accept valid User-Agent with space and email', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: 'MyCompany contact@example.com'
|
||||
});
|
||||
|
||||
expect(provider).toBeInstanceOf(opendata.SecEdgarProvider);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - Availability', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
await tap.test('should report as available', async () => {
|
||||
const isAvailable = await provider.isAvailable();
|
||||
expect(isAvailable).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - Fetch Fundamentals', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT,
|
||||
timeout: 30000
|
||||
});
|
||||
|
||||
await tap.test('should fetch fundamentals for Apple (AAPL)', async () => {
|
||||
const fundamentals = await provider.fetchData({
|
||||
type: 'fundamentals-current',
|
||||
ticker: TEST_TICKER
|
||||
});
|
||||
|
||||
// Verify structure
|
||||
expect(fundamentals).toBeDefined();
|
||||
expect(fundamentals).not.toBeInstanceOf(Array);
|
||||
|
||||
const data = fundamentals as opendata.IStockFundamentals;
|
||||
|
||||
// Basic fields
|
||||
expect(data.ticker).toEqual('AAPL');
|
||||
expect(data.cik).toBeDefined();
|
||||
expect(data.companyName).toEqual('Apple Inc.');
|
||||
expect(data.provider).toEqual('SEC EDGAR');
|
||||
expect(data.timestamp).toBeInstanceOf(Date);
|
||||
expect(data.fetchedAt).toBeInstanceOf(Date);
|
||||
|
||||
// Financial metrics (Apple should have all of these)
|
||||
expect(data.earningsPerShareDiluted).toBeDefined();
|
||||
expect(data.earningsPerShareDiluted).toBeGreaterThan(0);
|
||||
|
||||
expect(data.sharesOutstanding).toBeDefined();
|
||||
expect(data.sharesOutstanding).toBeGreaterThan(0);
|
||||
|
||||
expect(data.revenue).toBeDefined();
|
||||
expect(data.revenue).toBeGreaterThan(0);
|
||||
|
||||
expect(data.netIncome).toBeDefined();
|
||||
expect(data.assets).toBeDefined();
|
||||
expect(data.assets).toBeGreaterThan(0);
|
||||
|
||||
expect(data.liabilities).toBeDefined();
|
||||
expect(data.stockholdersEquity).toBeDefined();
|
||||
|
||||
// Metadata
|
||||
expect(data.fiscalYear).toBeDefined();
|
||||
|
||||
console.log('\n📊 Sample Apple Fundamentals:');
|
||||
console.log(` Company: ${data.companyName} (CIK: ${data.cik})`);
|
||||
console.log(` EPS (Diluted): $${data.earningsPerShareDiluted?.toFixed(2)}`);
|
||||
console.log(` Shares Outstanding: ${(data.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
|
||||
console.log(` Revenue: $${(data.revenue! / 1_000_000_000).toFixed(2)}B`);
|
||||
console.log(` Net Income: $${(data.netIncome! / 1_000_000_000).toFixed(2)}B`);
|
||||
console.log(` Assets: $${(data.assets! / 1_000_000_000).toFixed(2)}B`);
|
||||
console.log(` Fiscal Year: ${data.fiscalYear}`);
|
||||
});
|
||||
|
||||
await tap.test('should throw error for invalid ticker', async () => {
|
||||
try {
|
||||
await provider.fetchData({
|
||||
type: 'fundamentals-current',
|
||||
ticker: 'INVALIDTICKER123456'
|
||||
});
|
||||
throw new Error('Should have thrown error');
|
||||
} catch (error) {
|
||||
expect(error.message).toContain('CIK not found');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - Batch Fetch', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT,
|
||||
timeout: 30000
|
||||
});
|
||||
|
||||
await tap.test('should fetch fundamentals for multiple tickers', async () => {
|
||||
const result = await provider.fetchData({
|
||||
type: 'fundamentals-batch',
|
||||
tickers: ['AAPL', 'MSFT']
|
||||
});
|
||||
|
||||
expect(result).toBeInstanceOf(Array);
|
||||
|
||||
const fundamentalsList = result as opendata.IStockFundamentals[];
|
||||
expect(fundamentalsList.length).toEqual(2);
|
||||
|
||||
// Check Apple
|
||||
const apple = fundamentalsList.find(f => f.ticker === 'AAPL');
|
||||
expect(apple).toBeDefined();
|
||||
expect(apple!.companyName).toEqual('Apple Inc.');
|
||||
expect(apple!.earningsPerShareDiluted).toBeGreaterThan(0);
|
||||
|
||||
// Check Microsoft
|
||||
const microsoft = fundamentalsList.find(f => f.ticker === 'MSFT');
|
||||
expect(microsoft).toBeDefined();
|
||||
expect(microsoft!.companyName).toContain('Microsoft');
|
||||
expect(microsoft!.earningsPerShareDiluted).toBeGreaterThan(0);
|
||||
|
||||
console.log('\n📊 Batch Fetch Results:');
|
||||
fundamentalsList.forEach(f => {
|
||||
console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - CIK Caching', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
await tap.test('should cache CIK lookups', async () => {
|
||||
// Clear cache first
|
||||
provider.clearCache();
|
||||
|
||||
let stats = provider.getCacheStats();
|
||||
expect(stats.cikCacheSize).toEqual(0);
|
||||
|
||||
// First fetch (should populate cache)
|
||||
await provider.fetchData({
|
||||
type: 'fundamentals-current',
|
||||
ticker: 'AAPL'
|
||||
});
|
||||
|
||||
stats = provider.getCacheStats();
|
||||
expect(stats.cikCacheSize).toBeGreaterThan(0);
|
||||
|
||||
console.log(`\n💾 CIK Cache: ${stats.cikCacheSize} entries`);
|
||||
});
|
||||
|
||||
await tap.test('should clear cache', async () => {
|
||||
provider.clearCache();
|
||||
|
||||
const stats = provider.getCacheStats();
|
||||
expect(stats.cikCacheSize).toEqual(0);
|
||||
expect(stats.hasTickerList).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - Rate Limiting', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
await tap.test('should handle multiple rapid requests without exceeding rate limit', async () => {
|
||||
// Make 5 requests in succession
|
||||
// Rate limiter should ensure we don't exceed 10 req/sec
|
||||
const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'META'];
|
||||
const startTime = Date.now();
|
||||
|
||||
const promises = tickers.map(ticker =>
|
||||
provider.fetchData({
|
||||
type: 'fundamentals-current',
|
||||
ticker
|
||||
})
|
||||
);
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
expect(results.length).toEqual(5);
|
||||
console.log(`\n⏱️ 5 requests completed in ${duration}ms (avg: ${Math.round(duration / 5)}ms/request)`);
|
||||
|
||||
// Verify all results are valid
|
||||
results.forEach((result, index) => {
|
||||
expect(result).toBeDefined();
|
||||
const data = result as opendata.IStockFundamentals;
|
||||
expect(data.ticker).toEqual(tickers[index]);
|
||||
expect(data.earningsPerShareDiluted).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('SEC EDGAR Provider - Market Cap Calculation', async () => {
|
||||
const provider = new opendata.SecEdgarProvider({
|
||||
userAgent: TEST_USER_AGENT
|
||||
});
|
||||
|
||||
await tap.test('should provide data needed for market cap calculation', async () => {
|
||||
const fundamentals = await provider.fetchData({
|
||||
type: 'fundamentals-current',
|
||||
ticker: 'AAPL'
|
||||
}) as opendata.IStockFundamentals;
|
||||
|
||||
expect(fundamentals.sharesOutstanding).toBeDefined();
|
||||
expect(fundamentals.earningsPerShareDiluted).toBeDefined();
|
||||
|
||||
// Simulate current price (in real usage, this comes from price provider)
|
||||
const simulatedPrice = 270.37;
|
||||
|
||||
// Calculate market cap
|
||||
const marketCap = fundamentals.sharesOutstanding! * simulatedPrice;
|
||||
const pe = simulatedPrice / fundamentals.earningsPerShareDiluted!;
|
||||
|
||||
console.log('\n💰 Calculated Metrics (with simulated price $270.37):');
|
||||
console.log(` Shares Outstanding: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
|
||||
console.log(` Market Cap: $${(marketCap / 1_000_000_000_000).toFixed(2)}T`);
|
||||
console.log(` EPS: $${fundamentals.earningsPerShareDiluted!.toFixed(2)}`);
|
||||
console.log(` P/E Ratio: ${pe.toFixed(2)}`);
|
||||
|
||||
expect(marketCap).toBeGreaterThan(0);
|
||||
expect(pe).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@fin.cx/opendata',
|
||||
version: '3.0.0',
|
||||
version: '3.1.0',
|
||||
description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
|
||||
}
|
||||
|
||||
404
ts/stocks/classes.fundamentalsservice.ts
Normal file
404
ts/stocks/classes.fundamentalsservice.ts
Normal file
@@ -0,0 +1,404 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type {
|
||||
IFundamentalsProvider,
|
||||
IFundamentalsProviderConfig,
|
||||
IFundamentalsProviderRegistry,
|
||||
IStockFundamentals,
|
||||
IFundamentalsRequest
|
||||
} from './interfaces/fundamentals.js';
|
||||
|
||||
interface IProviderEntry {
|
||||
provider: IFundamentalsProvider;
|
||||
config: IFundamentalsProviderConfig;
|
||||
lastError?: Error;
|
||||
lastErrorTime?: Date;
|
||||
successCount: number;
|
||||
errorCount: number;
|
||||
}
|
||||
|
||||
interface ICacheEntry {
|
||||
fundamentals: IStockFundamentals | IStockFundamentals[];
|
||||
timestamp: Date;
|
||||
ttl: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Service for managing fundamental data providers and caching
|
||||
* Parallel to StockPriceService but for fundamental data instead of prices
|
||||
*/
|
||||
export class FundamentalsService implements IFundamentalsProviderRegistry {
|
||||
private providers = new Map<string, IProviderEntry>();
|
||||
private cache = new Map<string, ICacheEntry>();
|
||||
private logger = console;
|
||||
|
||||
private cacheConfig = {
|
||||
ttl: 90 * 24 * 60 * 60 * 1000, // 90 days default (fundamentals change quarterly)
|
||||
maxEntries: 10000
|
||||
};
|
||||
|
||||
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
|
||||
if (cacheConfig) {
|
||||
this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a fundamentals provider
|
||||
*/
|
||||
public register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void {
|
||||
const defaultConfig: IFundamentalsProviderConfig = {
|
||||
enabled: true,
|
||||
priority: provider.priority,
|
||||
timeout: 30000, // Longer timeout for fundamental data
|
||||
retryAttempts: 2,
|
||||
retryDelay: 1000,
|
||||
cacheTTL: this.cacheConfig.ttl
|
||||
};
|
||||
|
||||
const mergedConfig = { ...defaultConfig, ...config };
|
||||
|
||||
this.providers.set(provider.name, {
|
||||
provider,
|
||||
config: mergedConfig,
|
||||
successCount: 0,
|
||||
errorCount: 0
|
||||
});
|
||||
|
||||
console.log(`Registered fundamentals provider: ${provider.name}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister a provider
|
||||
*/
|
||||
public unregister(providerName: string): void {
|
||||
this.providers.delete(providerName);
|
||||
console.log(`Unregistered fundamentals provider: ${providerName}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific provider by name
|
||||
*/
|
||||
public getProvider(name: string): IFundamentalsProvider | undefined {
|
||||
return this.providers.get(name)?.provider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all registered providers
|
||||
*/
|
||||
public getAllProviders(): IFundamentalsProvider[] {
|
||||
return Array.from(this.providers.values()).map(entry => entry.provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get enabled providers sorted by priority
|
||||
*/
|
||||
public getEnabledProviders(): IFundamentalsProvider[] {
|
||||
return Array.from(this.providers.values())
|
||||
.filter(entry => entry.config.enabled)
|
||||
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
|
||||
.map(entry => entry.provider);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get fundamental data for a single ticker
|
||||
*/
|
||||
public async getFundamentals(ticker: string): Promise<IStockFundamentals> {
|
||||
const result = await this.getData({
|
||||
type: 'fundamentals-current',
|
||||
ticker
|
||||
});
|
||||
return result as IStockFundamentals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get fundamental data for multiple tickers
|
||||
*/
|
||||
public async getBatchFundamentals(tickers: string[]): Promise<IStockFundamentals[]> {
|
||||
const result = await this.getData({
|
||||
type: 'fundamentals-batch',
|
||||
tickers
|
||||
});
|
||||
return result as IStockFundamentals[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified data fetching method
|
||||
*/
|
||||
public async getData(
|
||||
request: IFundamentalsRequest
|
||||
): Promise<IStockFundamentals | IStockFundamentals[]> {
|
||||
const cacheKey = this.getCacheKey(request);
|
||||
const cached = this.getFromCache(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
console.log(`Cache hit for ${this.getRequestDescription(request)}`);
|
||||
return cached;
|
||||
}
|
||||
|
||||
const providers = this.getEnabledProviders();
|
||||
if (providers.length === 0) {
|
||||
throw new Error('No fundamentals providers available');
|
||||
}
|
||||
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (const provider of providers) {
|
||||
const entry = this.providers.get(provider.name)!;
|
||||
|
||||
try {
|
||||
const result = await this.fetchWithRetry(
|
||||
() => provider.fetchData(request),
|
||||
entry.config
|
||||
);
|
||||
|
||||
entry.successCount++;
|
||||
|
||||
// Use provider-specific cache TTL or default
|
||||
const ttl = entry.config.cacheTTL || this.cacheConfig.ttl;
|
||||
this.addToCache(cacheKey, result, ttl);
|
||||
|
||||
console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
entry.errorCount++;
|
||||
entry.lastError = error as Error;
|
||||
entry.lastErrorTime = new Date();
|
||||
lastError = error as Error;
|
||||
|
||||
console.warn(
|
||||
`Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enrich fundamentals with calculated metrics using current price
|
||||
*/
|
||||
public async enrichWithPrice(
|
||||
fundamentals: IStockFundamentals,
|
||||
price: number
|
||||
): Promise<IStockFundamentals> {
|
||||
const enriched = { ...fundamentals };
|
||||
|
||||
// Calculate market cap: price × shares outstanding
|
||||
if (fundamentals.sharesOutstanding) {
|
||||
enriched.marketCap = price * fundamentals.sharesOutstanding;
|
||||
}
|
||||
|
||||
// Calculate P/E ratio: price / EPS
|
||||
if (fundamentals.earningsPerShareDiluted && fundamentals.earningsPerShareDiluted > 0) {
|
||||
enriched.priceToEarnings = price / fundamentals.earningsPerShareDiluted;
|
||||
}
|
||||
|
||||
// Calculate price-to-book: market cap / stockholders equity
|
||||
if (enriched.marketCap && fundamentals.stockholdersEquity && fundamentals.stockholdersEquity > 0) {
|
||||
enriched.priceToBook = enriched.marketCap / fundamentals.stockholdersEquity;
|
||||
}
|
||||
|
||||
return enriched;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enrich batch fundamentals with prices
|
||||
*/
|
||||
public async enrichBatchWithPrices(
|
||||
fundamentalsList: IStockFundamentals[],
|
||||
priceMap: Map<string, number>
|
||||
): Promise<IStockFundamentals[]> {
|
||||
return Promise.all(
|
||||
fundamentalsList.map(fundamentals => {
|
||||
const price = priceMap.get(fundamentals.ticker);
|
||||
if (price) {
|
||||
return this.enrichWithPrice(fundamentals, price);
|
||||
}
|
||||
return Promise.resolve(fundamentals);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check health of all providers
|
||||
*/
|
||||
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
|
||||
const health = new Map<string, boolean>();
|
||||
|
||||
for (const [name, entry] of this.providers) {
|
||||
if (!entry.config.enabled) {
|
||||
health.set(name, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const isAvailable = await entry.provider.isAvailable();
|
||||
health.set(name, isAvailable);
|
||||
} catch (error) {
|
||||
health.set(name, false);
|
||||
console.error(`Health check failed for ${name}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return health;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get provider statistics
|
||||
*/
|
||||
public getProviderStats(): Map<
|
||||
string,
|
||||
{
|
||||
successCount: number;
|
||||
errorCount: number;
|
||||
lastError?: string;
|
||||
lastErrorTime?: Date;
|
||||
}
|
||||
> {
|
||||
const stats = new Map();
|
||||
|
||||
for (const [name, entry] of this.providers) {
|
||||
stats.set(name, {
|
||||
successCount: entry.successCount,
|
||||
errorCount: entry.errorCount,
|
||||
lastError: entry.lastError?.message,
|
||||
lastErrorTime: entry.lastErrorTime
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached data
|
||||
*/
|
||||
public clearCache(): void {
|
||||
this.cache.clear();
|
||||
console.log('Fundamentals cache cleared');
|
||||
}
|
||||
|
||||
/**
|
||||
* Set cache TTL
|
||||
*/
|
||||
public setCacheTTL(ttl: number): void {
|
||||
this.cacheConfig.ttl = ttl;
|
||||
console.log(`Fundamentals cache TTL set to ${ttl}ms`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
public getCacheStats(): {
|
||||
size: number;
|
||||
maxEntries: number;
|
||||
ttl: number;
|
||||
} {
|
||||
return {
|
||||
size: this.cache.size,
|
||||
maxEntries: this.cacheConfig.maxEntries,
|
||||
ttl: this.cacheConfig.ttl
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch with retry logic
|
||||
*/
|
||||
private async fetchWithRetry<T>(
|
||||
fetchFn: () => Promise<T>,
|
||||
config: IFundamentalsProviderConfig
|
||||
): Promise<T> {
|
||||
const maxAttempts = config.retryAttempts || 1;
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fetchFn();
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
if (attempt < maxAttempts) {
|
||||
const delay = (config.retryDelay || 1000) * attempt;
|
||||
console.log(`Retry attempt ${attempt} after ${delay}ms`);
|
||||
await plugins.smartdelay.delayFor(delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Unknown error during fetch');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key for request
|
||||
*/
|
||||
private getCacheKey(request: IFundamentalsRequest): string {
|
||||
switch (request.type) {
|
||||
case 'fundamentals-current':
|
||||
return `fundamentals:${request.ticker}`;
|
||||
case 'fundamentals-batch':
|
||||
const tickers = request.tickers.sort().join(',');
|
||||
return `fundamentals-batch:${tickers}`;
|
||||
default:
|
||||
return `unknown:${JSON.stringify(request)}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get from cache if not expired
|
||||
*/
|
||||
private getFromCache(key: string): IStockFundamentals | IStockFundamentals[] | null {
|
||||
const entry = this.cache.get(key);
|
||||
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if cache entry has expired
|
||||
const age = Date.now() - entry.timestamp.getTime();
|
||||
if (entry.ttl !== Infinity && age > entry.ttl) {
|
||||
this.cache.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.fundamentals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to cache with TTL
|
||||
*/
|
||||
private addToCache(
|
||||
key: string,
|
||||
fundamentals: IStockFundamentals | IStockFundamentals[],
|
||||
ttl?: number
|
||||
): void {
|
||||
// Enforce max entries limit
|
||||
if (this.cache.size >= this.cacheConfig.maxEntries) {
|
||||
// Remove oldest entry
|
||||
const oldestKey = this.cache.keys().next().value;
|
||||
if (oldestKey) {
|
||||
this.cache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
fundamentals,
|
||||
timestamp: new Date(),
|
||||
ttl: ttl || this.cacheConfig.ttl
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable request description
|
||||
*/
|
||||
private getRequestDescription(request: IFundamentalsRequest): string {
|
||||
switch (request.type) {
|
||||
case 'fundamentals-current':
|
||||
return `fundamentals for ${request.ticker}`;
|
||||
case 'fundamentals-batch':
|
||||
return `fundamentals for ${request.tickers.length} tickers`;
|
||||
default:
|
||||
return 'fundamentals data';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
// Export all interfaces
|
||||
export * from './interfaces/stockprice.js';
|
||||
export * from './interfaces/provider.js';
|
||||
export * from './interfaces/fundamentals.js';
|
||||
|
||||
// Export main service
|
||||
// Export main services
|
||||
export * from './classes.stockservice.js';
|
||||
export * from './classes.fundamentalsservice.js';
|
||||
|
||||
// Export providers
|
||||
export * from './providers/provider.yahoo.js';
|
||||
export * from './providers/provider.marketstack.js';
|
||||
export * from './providers/provider.marketstack.js';
|
||||
export * from './providers/provider.secedgar.js';
|
||||
102
ts/stocks/interfaces/fundamentals.ts
Normal file
102
ts/stocks/interfaces/fundamentals.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* Interfaces for stock fundamental data (financials from SEC filings)
|
||||
* Separate from stock price data (OHLCV) to maintain clean architecture
|
||||
*/
|
||||
|
||||
// Request types for fundamental data
|
||||
export interface IFundamentalsCurrentRequest {
|
||||
type: 'fundamentals-current';
|
||||
ticker: string;
|
||||
}
|
||||
|
||||
export interface IFundamentalsBatchRequest {
|
||||
type: 'fundamentals-batch';
|
||||
tickers: string[];
|
||||
}
|
||||
|
||||
export type IFundamentalsRequest =
|
||||
| IFundamentalsCurrentRequest
|
||||
| IFundamentalsBatchRequest;
|
||||
|
||||
/**
|
||||
* Stock fundamental data from SEC filings (10-K, 10-Q)
|
||||
* Contains financial metrics like EPS, Revenue, Assets, etc.
|
||||
*/
|
||||
export interface IStockFundamentals {
|
||||
ticker: string;
|
||||
cik: string;
|
||||
companyName: string;
|
||||
provider: string;
|
||||
timestamp: Date;
|
||||
fetchedAt: Date;
|
||||
|
||||
// Per-share metrics
|
||||
earningsPerShareBasic?: number;
|
||||
earningsPerShareDiluted?: number;
|
||||
sharesOutstanding?: number;
|
||||
weightedAverageSharesOutstanding?: number;
|
||||
|
||||
// Income statement (annual USD)
|
||||
revenue?: number;
|
||||
netIncome?: number;
|
||||
operatingIncome?: number;
|
||||
grossProfit?: number;
|
||||
costOfRevenue?: number;
|
||||
|
||||
// Balance sheet (annual USD)
|
||||
assets?: number;
|
||||
liabilities?: number;
|
||||
stockholdersEquity?: number;
|
||||
cash?: number;
|
||||
propertyPlantEquipment?: number;
|
||||
|
||||
// Calculated metrics (requires current price)
|
||||
marketCap?: number; // price × sharesOutstanding
|
||||
priceToEarnings?: number; // price / EPS
|
||||
priceToBook?: number; // marketCap / stockholdersEquity
|
||||
|
||||
// Metadata
|
||||
fiscalYear?: string;
|
||||
fiscalQuarter?: string;
|
||||
filingDate?: Date;
|
||||
form?: '10-K' | '10-Q' | string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider interface for fetching fundamental data
|
||||
* Parallel to IStockProvider but for fundamentals instead of prices
|
||||
*/
|
||||
export interface IFundamentalsProvider {
|
||||
name: string;
|
||||
priority: number;
|
||||
fetchData(request: IFundamentalsRequest): Promise<IStockFundamentals | IStockFundamentals[]>;
|
||||
isAvailable(): Promise<boolean>;
|
||||
readonly requiresAuth: boolean;
|
||||
readonly rateLimit?: {
|
||||
requestsPerMinute: number;
|
||||
requestsPerDay?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for fundamentals providers
|
||||
*/
|
||||
export interface IFundamentalsProviderConfig {
|
||||
enabled: boolean;
|
||||
priority?: number;
|
||||
timeout?: number;
|
||||
retryAttempts?: number;
|
||||
retryDelay?: number;
|
||||
cacheTTL?: number; // Custom cache TTL for this provider
|
||||
}
|
||||
|
||||
/**
|
||||
* Registry for managing fundamental data providers
|
||||
*/
|
||||
export interface IFundamentalsProviderRegistry {
|
||||
register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void;
|
||||
unregister(providerName: string): void;
|
||||
getProvider(name: string): IFundamentalsProvider | undefined;
|
||||
getAllProviders(): IFundamentalsProvider[];
|
||||
getEnabledProviders(): IFundamentalsProvider[];
|
||||
}
|
||||
429
ts/stocks/providers/provider.secedgar.ts
Normal file
429
ts/stocks/providers/provider.secedgar.ts
Normal file
@@ -0,0 +1,429 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type {
|
||||
IFundamentalsProvider,
|
||||
IStockFundamentals,
|
||||
IFundamentalsRequest,
|
||||
IFundamentalsCurrentRequest,
|
||||
IFundamentalsBatchRequest
|
||||
} from '../interfaces/fundamentals.js';
|
||||
|
||||
/**
|
||||
* Configuration for SEC EDGAR provider
|
||||
*/
|
||||
export interface ISecEdgarConfig {
|
||||
userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx")
|
||||
cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change)
|
||||
fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings)
|
||||
timeout?: number; // Request timeout in ms
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate limiter for SEC EDGAR API
|
||||
* SEC requires: 10 requests per second maximum
|
||||
*/
|
||||
class RateLimiter {
|
||||
private requestTimes: number[] = [];
|
||||
private readonly maxRequestsPerSecond = 10;
|
||||
|
||||
public async waitForSlot(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const oneSecondAgo = now - 1000;
|
||||
|
||||
// Remove requests older than 1 second
|
||||
this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo);
|
||||
|
||||
// If we've hit the limit, wait
|
||||
if (this.requestTimes.length >= this.maxRequestsPerSecond) {
|
||||
const oldestRequest = this.requestTimes[0];
|
||||
const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer
|
||||
await plugins.smartdelay.delayFor(waitTime);
|
||||
return this.waitForSlot(); // Recursively check again
|
||||
}
|
||||
|
||||
// Record this request
|
||||
this.requestTimes.push(now);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SEC EDGAR Fundamental Data Provider
|
||||
*
|
||||
* Features:
|
||||
* - Free public access (no API key required)
|
||||
* - All US public companies
|
||||
* - Financial data from 10-K/10-Q filings
|
||||
* - US GAAP standardized metrics
|
||||
* - Historical data back to ~2009
|
||||
* - < 1 minute filing delay
|
||||
*
|
||||
* Documentation: https://www.sec.gov/edgar/sec-api-documentation
|
||||
*
|
||||
* Rate Limits:
|
||||
* - 10 requests per second (enforced by SEC)
|
||||
* - Requires User-Agent header in format: "Company Name Email"
|
||||
*
|
||||
* Data Sources:
|
||||
* - Company Facts API: /api/xbrl/companyfacts/CIK##########.json
|
||||
* - Ticker Lookup: /files/company_tickers.json
|
||||
*/
|
||||
export class SecEdgarProvider implements IFundamentalsProvider {
|
||||
public name = 'SEC EDGAR';
|
||||
public priority = 100; // High priority - free, authoritative, comprehensive
|
||||
public readonly requiresAuth = false; // No API key needed!
|
||||
public readonly rateLimit = {
|
||||
requestsPerMinute: 600, // 10 requests/second = 600/minute
|
||||
requestsPerDay: undefined // No daily limit
|
||||
};
|
||||
|
||||
private logger = console;
|
||||
private baseUrl = 'https://data.sec.gov/api/xbrl';
|
||||
private tickersUrl = 'https://www.sec.gov/files/company_tickers.json';
|
||||
private userAgent: string;
|
||||
private config: Required<ISecEdgarConfig>;
|
||||
|
||||
// Caching
|
||||
private cikCache = new Map<string, { cik: string; timestamp: Date }>();
|
||||
private tickerListCache: { data: any; timestamp: Date } | null = null;
|
||||
|
||||
// Rate limiting
|
||||
private rateLimiter = new RateLimiter();
|
||||
|
||||
constructor(config: ISecEdgarConfig) {
|
||||
// Validate User-Agent
|
||||
if (!config.userAgent) {
|
||||
throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")');
|
||||
}
|
||||
|
||||
// Validate User-Agent format (must contain space and @ symbol)
|
||||
if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) {
|
||||
throw new Error(
|
||||
'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")'
|
||||
);
|
||||
}
|
||||
|
||||
this.userAgent = config.userAgent;
|
||||
this.config = {
|
||||
userAgent: config.userAgent,
|
||||
cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days
|
||||
fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days
|
||||
timeout: config.timeout || 30000
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified data fetching method
|
||||
*/
|
||||
public async fetchData(
|
||||
request: IFundamentalsRequest
|
||||
): Promise<IStockFundamentals | IStockFundamentals[]> {
|
||||
switch (request.type) {
|
||||
case 'fundamentals-current':
|
||||
return this.fetchFundamentals(request);
|
||||
case 'fundamentals-batch':
|
||||
return this.fetchBatchFundamentals(request);
|
||||
default:
|
||||
throw new Error(`Unsupported request type: ${(request as any).type}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch fundamental data for a single ticker
|
||||
*/
|
||||
private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise<IStockFundamentals> {
|
||||
try {
|
||||
// 1. Get CIK for ticker (with caching)
|
||||
const cik = await this.getCIK(request.ticker);
|
||||
|
||||
// 2. Fetch company facts from SEC (with rate limiting)
|
||||
const companyFacts = await this.fetchCompanyFacts(cik);
|
||||
|
||||
// 3. Parse facts into structured fundamental data
|
||||
return this.parseCompanyFacts(request.ticker, cik, companyFacts);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error);
|
||||
throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch fundamental data for multiple tickers
|
||||
*/
|
||||
private async fetchBatchFundamentals(
|
||||
request: IFundamentalsBatchRequest
|
||||
): Promise<IStockFundamentals[]> {
|
||||
const results: IStockFundamentals[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const ticker of request.tickers) {
|
||||
try {
|
||||
const fundamentals = await this.fetchFundamentals({
|
||||
type: 'fundamentals-current',
|
||||
ticker
|
||||
});
|
||||
results.push(fundamentals);
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error);
|
||||
errors.push(`${ticker}: ${error.message}`);
|
||||
// Continue with other tickers
|
||||
}
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get CIK (Central Index Key) for a ticker symbol
|
||||
* Uses SEC's public ticker-to-CIK mapping file
|
||||
*/
|
||||
private async getCIK(ticker: string): Promise<string> {
|
||||
const tickerUpper = ticker.toUpperCase();
|
||||
|
||||
// Check cache first
|
||||
const cached = this.cikCache.get(tickerUpper);
|
||||
if (cached) {
|
||||
const age = Date.now() - cached.timestamp.getTime();
|
||||
if (age < this.config.cikCacheTTL) {
|
||||
return cached.cik;
|
||||
}
|
||||
// Cache expired, remove it
|
||||
this.cikCache.delete(tickerUpper);
|
||||
}
|
||||
|
||||
// Fetch ticker list (with caching at list level)
|
||||
const tickers = await this.fetchTickerList();
|
||||
|
||||
// Find ticker in list (case-insensitive)
|
||||
const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper);
|
||||
|
||||
if (!entry) {
|
||||
throw new Error(`CIK not found for ticker ${ticker}`);
|
||||
}
|
||||
|
||||
const cik = String((entry as any).cik_str);
|
||||
|
||||
// Cache the result
|
||||
this.cikCache.set(tickerUpper, {
|
||||
cik,
|
||||
timestamp: new Date()
|
||||
});
|
||||
|
||||
return cik;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the SEC ticker-to-CIK mapping list
|
||||
* Cached for 24 hours (list updates daily)
|
||||
*/
|
||||
private async fetchTickerList(): Promise<any> {
|
||||
// Check cache
|
||||
if (this.tickerListCache) {
|
||||
const age = Date.now() - this.tickerListCache.timestamp.getTime();
|
||||
if (age < 24 * 60 * 60 * 1000) { // 24 hours
|
||||
return this.tickerListCache.data;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for rate limit slot
|
||||
await this.rateLimiter.waitForSlot();
|
||||
|
||||
// Fetch from SEC
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(this.tickersUrl)
|
||||
.headers({
|
||||
'User-Agent': this.userAgent,
|
||||
'Accept': 'application/json'
|
||||
})
|
||||
.timeout(this.config.timeout)
|
||||
.get();
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Cache the list
|
||||
this.tickerListCache = {
|
||||
data,
|
||||
timestamp: new Date()
|
||||
};
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch company facts from SEC EDGAR
|
||||
*/
|
||||
private async fetchCompanyFacts(cik: string): Promise<any> {
|
||||
// Pad CIK to 10 digits
|
||||
const paddedCIK = cik.padStart(10, '0');
|
||||
const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`;
|
||||
|
||||
// Wait for rate limit slot
|
||||
await this.rateLimiter.waitForSlot();
|
||||
|
||||
// Fetch from SEC
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.headers({
|
||||
'User-Agent': this.userAgent,
|
||||
'Accept': 'application/json',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Host': 'data.sec.gov'
|
||||
})
|
||||
.timeout(this.config.timeout)
|
||||
.get();
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Validate response
|
||||
if (!data || !data.facts) {
|
||||
throw new Error('Invalid response from SEC EDGAR API');
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse SEC company facts into structured fundamental data
|
||||
*/
|
||||
private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals {
|
||||
const usGaap = data.facts?.['us-gaap'];
|
||||
|
||||
if (!usGaap) {
|
||||
throw new Error('No US GAAP data available');
|
||||
}
|
||||
|
||||
// Extract latest values for key metrics
|
||||
const fundamentals: IStockFundamentals = {
|
||||
ticker: ticker.toUpperCase(),
|
||||
cik: cik,
|
||||
companyName: data.entityName,
|
||||
provider: this.name,
|
||||
timestamp: new Date(),
|
||||
fetchedAt: new Date(),
|
||||
|
||||
// Per-share metrics
|
||||
earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'),
|
||||
earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'),
|
||||
sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'),
|
||||
weightedAverageSharesOutstanding: this.getLatestValue(
|
||||
usGaap,
|
||||
'WeightedAverageNumberOfSharesOutstandingBasic'
|
||||
),
|
||||
|
||||
// Income statement
|
||||
revenue: this.getLatestValue(usGaap, 'Revenues') ||
|
||||
this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'),
|
||||
netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'),
|
||||
operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'),
|
||||
grossProfit: this.getLatestValue(usGaap, 'GrossProfit'),
|
||||
costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'),
|
||||
|
||||
// Balance sheet
|
||||
assets: this.getLatestValue(usGaap, 'Assets'),
|
||||
liabilities: this.getLatestValue(usGaap, 'Liabilities'),
|
||||
stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'),
|
||||
cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'),
|
||||
propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'),
|
||||
|
||||
// Metadata (from latest available data point)
|
||||
fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy,
|
||||
fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp,
|
||||
filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed
|
||||
? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed)
|
||||
: undefined,
|
||||
form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form
|
||||
};
|
||||
|
||||
return fundamentals;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest value for a US GAAP metric
|
||||
*/
|
||||
private getLatestValue(usGaap: any, metricName: string): number | undefined {
|
||||
const metric = usGaap[metricName];
|
||||
if (!metric?.units) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Get the first unit type (USD, shares, etc.)
|
||||
const unitType = Object.keys(metric.units)[0];
|
||||
const values = metric.units[unitType];
|
||||
|
||||
if (!values || !Array.isArray(values) || values.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Get the latest value (last in array)
|
||||
const latest = values[values.length - 1];
|
||||
return latest?.val;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata from the latest data point
|
||||
*/
|
||||
private getLatestMetadata(usGaap: any, metricName: string): any | undefined {
|
||||
const metric = usGaap[metricName];
|
||||
if (!metric?.units) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const unitType = Object.keys(metric.units)[0];
|
||||
const values = metric.units[unitType];
|
||||
|
||||
if (!values || !Array.isArray(values) || values.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return values[values.length - 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if SEC EDGAR API is available
|
||||
*/
|
||||
public async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
// Test with Apple's well-known CIK
|
||||
const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`;
|
||||
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.headers({
|
||||
'User-Agent': this.userAgent,
|
||||
'Accept': 'application/json'
|
||||
})
|
||||
.timeout(5000)
|
||||
.get();
|
||||
|
||||
const data = await response.json();
|
||||
return data && data.facts !== undefined;
|
||||
} catch (error) {
|
||||
this.logger.warn('SEC EDGAR provider is not available:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
public getCacheStats(): {
|
||||
cikCacheSize: number;
|
||||
hasTickerList: boolean;
|
||||
} {
|
||||
return {
|
||||
cikCacheSize: this.cikCache.size,
|
||||
hasTickerList: this.tickerListCache !== null
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all caches
|
||||
*/
|
||||
public clearCache(): void {
|
||||
this.cikCache.clear();
|
||||
this.tickerListCache = null;
|
||||
this.logger.log('SEC EDGAR cache cleared');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user