45 Commits

Author SHA1 Message Date
3be2f0b855 v3.5.0
Some checks failed
Default (tags) / security (push) Failing after 22s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-07 08:05:59 +00:00
c38f895a72 feat(stocks): Add provider fetch limits, intraday incremental fetch, cache deduplication, and provider safety/warning improvements 2025-11-07 08:05:59 +00:00
27417d81bf v3.4.0
Some checks failed
Default (tags) / security (push) Failing after 28s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-06 22:58:46 +00:00
d80bbacb08 feat(stocks): Introduce unified stock data service, new providers, improved caching and German business data tooling 2025-11-06 22:58:46 +00:00
909b30117b 3.3.0
Some checks failed
Default (tags) / security (push) Failing after 23s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-02 16:34:24 +00:00
47fd770e48 feat(stocks/CoinGeckoProvider): Add CoinGecko provider for cryptocurrency prices, export and tests, and update documentation 2025-11-02 16:34:24 +00:00
fdea1bb149 3.2.2
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-01 15:20:20 +00:00
8286c30edf fix(handelsregister): Correct screenshot path handling in HandelsRegister and add local tool permissions 2025-11-01 15:20:20 +00:00
ea76776ee1 3.2.1
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 12s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-01 14:54:04 +00:00
54818293a1 fix(stocks/providers/provider.secedgar): Improve SEC EDGAR provider networking and error handling, update plugin path import, bump dev deps and add/refresh tests and lockfile 2025-11-01 14:54:04 +00:00
d49a738880 3.2.0
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-01 12:35:53 +00:00
6273faa2f9 feat(StockDataService): Add unified StockDataService and BaseProviderService with new stockdata interfaces, provider integrations, tests and README updates 2025-11-01 12:35:53 +00:00
d33c7e0f52 3.1.0
Some checks failed
Default (tags) / security (push) Failing after 23s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-11-01 11:59:23 +00:00
79930c40ac feat(fundamentals): Add FundamentalsService and SEC EDGAR provider with caching, rate-limiting, tests, and docs updates 2025-11-01 11:59:23 +00:00
448278243e 3.0.0
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 12s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-31 15:05:48 +00:00
ec3e4dde75 BREAKING CHANGE(stocks): Unify stock provider API to discriminated IStockDataRequest and add company name/fullname enrichment 2025-10-31 15:05:48 +00:00
596be63554 2.1.0
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 12s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-31 14:00:59 +00:00
8632f0e94b feat(stocks): Add unified stock data API (getData) with historical/OHLCV support, smart caching and provider enhancements 2025-10-31 14:00:59 +00:00
28ae2bd737 2.0.0
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 12s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-31 12:12:29 +00:00
c806524e0c BREAKING CHANGE(OpenData): Require explicit directory paths for OpenData (nogit/download/germanBusinessData); remove automatic .nogit creation; update HandelsRegister, JsonlDataProcessor, tests and README. 2025-10-31 12:12:29 +00:00
fea83153ba 1.7.0
Some checks failed
Default (tags) / security (push) Failing after 22s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-11 15:21:57 +00:00
4716ef03ba feat(stocks): Add Marketstack provider (EOD) with tests, exports and documentation updates 2025-10-11 15:21:57 +00:00
3b76de0831 1.6.1
Some checks failed
Default (tags) / security (push) Failing after 24s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-09-24 07:56:53 +00:00
e94a6f8d5b fix(stocks): Fix Yahoo provider request handling and bump dependency versions 2025-09-24 07:56:53 +00:00
1b1324d0f9 1.6.0
Some checks failed
Default (tags) / security (push) Failing after 29s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-07-13 15:08:20 +00:00
71a5a32198 feat(readme): Revamp documentation and package description for enhanced clarity 2025-07-13 15:08:19 +00:00
3dbf194320 1.5.4
Some checks failed
Default (tags) / security (push) Failing after 31s
Default (tags) / test (push) Failing after 15s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-07-11 09:14:09 +00:00
a29a50e825 update 2025-07-11 09:09:13 +00:00
daeff1ce93 update 2025-07-11 08:38:48 +00:00
298172c00b update 2025-07-08 15:39:29 +00:00
df677b38fb update 2025-07-08 15:30:13 +00:00
c344e47ae6 1.5.3
Some checks failed
Default (tags) / security (push) Failing after 17s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-09 15:55:29 +00:00
209af50a4c fix(test): Await file writes in Handelsregister tests to ensure all downloads complete before test end 2025-04-09 15:55:29 +00:00
1d0d44dc29 1.5.2
Some checks failed
Default (tags) / security (push) Failing after 18s
Default (tags) / test (push) Failing after 9s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-09 08:40:23 +00:00
c4d6403721 fix(readme): Improve .env configuration code block formatting in documentation 2025-04-09 08:40:23 +00:00
84cab94beb 1.5.1
Some checks failed
Default (tags) / security (push) Failing after 8s
Default (tags) / test (push) Failing after 8s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-09 06:28:27 +00:00
d17683cd67 fix(core): No changes detected in project files or documentation. This commit is a placeholder to record that nothing was updated. 2025-04-09 06:28:27 +00:00
39537c0568 1.5.0
Some checks failed
Default (tags) / security (push) Failing after 18s
Default (tags) / test (push) Failing after 8s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-09 06:26:52 +00:00
b768b67641 feat(documentation): Enhance project metadata and documentation with comprehensive usage examples, updated descriptions, and improved keywords. 2025-04-09 06:26:52 +00:00
25147deb7f 1.4.6
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 9s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-08 15:31:29 +00:00
4030bef7a8 fix(tests & jsonl): Improve test structure and refine JSONL parsing for incomplete data 2025-04-08 15:31:29 +00:00
c6964f0310 1.4.5
Some checks failed
Default (tags) / security (push) Failing after 8s
Default (tags) / test (push) Failing after 8s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-05 09:05:08 +00:00
9a9f203af2 fix(metadata): Update repository, bugs, and homepage URLs to code.foss.global 2025-04-05 09:05:08 +00:00
174086defc 1.4.4
Some checks failed
Default (tags) / security (push) Failing after 19s
Default (tags) / test (push) Failing after 8s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-05 09:02:32 +00:00
43c9d3b3b6 fix(dependencies & tests): Update dependency versions and refine test search query 2025-04-05 09:02:32 +00:00
41 changed files with 20553 additions and 4000 deletions

1
.serena/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/cache

67
.serena/project.yml Normal file
View File

@@ -0,0 +1,67 @@
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
# * For C, use cpp
# * For JavaScript, use typescript
# Special requirements:
# * csharp: Requires the presence of a .sln file in the project folder.
language: typescript
# whether to use the project's gitignore file to ignore files
# Added on 2025-04-07
ignore_all_files_in_gitignore: true
# list of additional paths to ignore
# same syntax as gitignore, so you can use * and **
# Was previously called `ignored_dirs`, please update your config if you are using that.
# Added (renamed) on 2025-04-07
ignored_paths: []
# whether the project is in read-only mode
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
# Added on 2025-04-18
read_only: false
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
# Below is the complete list of tools for convenience.
# To make sure you have the latest list of tools, and to view their descriptions,
# execute `uv run scripts/print_tool_overview.py`.
#
# * `activate_project`: Activates a project by name.
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
# * `create_text_file`: Creates/overwrites a file in the project directory.
# * `delete_lines`: Deletes a range of lines within a file.
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
# * `execute_shell_command`: Executes a shell command.
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
# * `initial_instructions`: Gets the initial instructions for the current project.
# Should only be used in settings where the system prompt cannot be set,
# e.g. in clients you have no control over, like Claude Desktop.
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
# * `insert_at_line`: Inserts content at a given line in a file.
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
# * `list_memories`: Lists memories in Serena's project-specific memory store.
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
# * `read_file`: Reads a file within the project directory.
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
# * `remove_project`: Removes a project from the Serena configuration.
# * `replace_lines`: Replaces a range of lines within a file with new content.
# * `replace_symbol_body`: Replaces the full definition of a symbol.
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
# * `search_for_pattern`: Performs a search for a pattern in the project.
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
# * `switch_modes`: Activates modes by providing a list of their names
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
excluded_tools: []
# initial prompt for the project. It will always be given to the LLM upon activating the project
# (contrary to the memories, which are loaded on demand).
initial_prompt: ""
project_name: "opendata"

View File

@@ -1,5 +1,183 @@
# Changelog
## 2025-11-07 - 3.5.0 - feat(stocks)
Add provider fetch limits, intraday incremental fetch, cache deduplication, and provider safety/warning improvements
- Add maxRecords and defaultIntradayLimit to IProviderConfig to control maximum records per request and default intraday limits.
- CoinGecko provider: enforce maxRecords when processing historical data, warn when large historical/intraday results are returned without explicit limits, preserve priority mappings when rebuilding the coin cache, and improve cache load logging.
- Marketstack provider: make safety maxRecords configurable, apply a configurable default intraday limit, warn when no explicit limit is provided, and ensure effective limits are applied to returned results.
- StockPriceService: always attempt incremental fetch for intraday requests without a date to fetch only new data since the last cached timestamp and fall back to full fetch when necessary.
- StockPriceService: deduplicate price arrays by timestamp before caching and after merges to avoid duplicate timestamps and reduce cache bloat.
- Introduce StockDataService for unified access to prices and fundamentals with automatic enrichment (market cap, P/E, price-to-book) and caching improvements.
- Various cache/TTL improvements and safer default behaviors for intraday, historical and live data to improve performance and memory usage.
## 2025-11-06 - 3.4.0 - feat(stocks)
Introduce unified stock data service, new providers, improved caching and German business data tooling
- Add StockDataService: unified API to fetch price + fundamentals with automatic enrichment and batch support
- Introduce BaseProviderService abstraction and refactor provider management, caching and retry logic
- Enhance StockPriceService: unified getData, discriminated union request types, data-type aware TTLs and smarter cache keys
- Add Marketstack provider with intraday/EOD selection, pagination, OHLCV and exchange filtering
- Add CoinGecko provider with robust rate-limiting, coin ID resolution and crypto support (current, historical, intraday)
- Add SEC EDGAR fundamentals provider: CIK lookup, company facts parsing, rate limiting and caching
- Improve FundamentalsService: unified fetching, caching and enrichment helpers (enrichWithPrice, enrichBatchWithPrices)
- Enhance Yahoo provider and other provider mappings for better company metadata and market state handling
- Add German business data tooling: JsonlDataProcessor for JSONL bulk imports, HandelsRegister browser automation with download handling and parsing
- Expose OpenData entry points: DB init, JSONL processing and Handelsregister integration; add readme/docs and usage examples
## 2025-11-02 - 3.3.0 - feat(stocks/CoinGeckoProvider)
Add CoinGecko provider for cryptocurrency prices, export and tests, and update documentation
- Implemented CoinGeckoProvider with rate limiting, coin-id resolution, and support for current, batch, historical and intraday price endpoints
- Added unit/integration tests for CoinGecko: test/test.coingecko.node+bun+deno.ts
- Exported CoinGeckoProvider from ts/stocks/index.ts
- Updated README and readme.hints.md with CoinGecko usage, provider notes and examples
- Added .claude/settings.local.json with webfetch and bash permissions required for testing and CI
## 2025-11-01 - 3.2.2 - fix(handelsregister)
Correct screenshot path handling in HandelsRegister and add local tool permissions
- ts/classes.handelsregister.ts: Replace string concatenation for screenshot path with a template literal and explicit string assertion to ensure the path is formed correctly for page.screenshot() and avoid type issues.
- Add .claude/settings.local.json: Introduce local Claude settings that grant specific tool permissions used during development and testing (bash commands, web fetches, pnpm build, tstest, etc.).
## 2025-11-01 - 3.2.1 - fix(stocks/providers/provider.secedgar)
Improve SEC EDGAR provider networking and error handling, update plugin path import, bump dev deps and add/refresh tests and lockfile
- SEC EDGAR provider: switch from SmartRequest to native fetch for ticker list and company facts, add AbortController-based timeouts, handle gzip automatically, improve response validation and error messages, and keep CIK/ticker-list caching
- Improve timeout and rate-limit handling in SecEdgarProvider (uses native fetch + explicit timeout clear), plus clearer logging on failures
- Update ts/plugins import to use node:path for Node compatibility
- Bump devDependencies: @git.zone/tsrun to ^1.6.2 and @git.zone/tstest to ^2.7.0; bump @push.rocks/smartrequest to ^4.3.4
- Add and refresh comprehensive test files (node/bun/deno variants) for fundamentals, marketstack, secedgar and stockdata services
- Add deno.lock (dependency lock) and a local .claude/settings.local.json for CI/permissions
## 2025-11-01 - 3.2.0 - feat(StockDataService)
Add unified StockDataService and BaseProviderService with new stockdata interfaces, provider integrations, tests and README updates
- Introduce StockDataService: unified API to fetch prices and fundamentals with automatic enrichment and caching
- Add IStockData and IStockDataServiceConfig interfaces to define combined price+fundamentals payloads and configuration
- Implement BaseProviderService abstraction to share provider registration, health, stats and caching logic
- Add classes.stockdataservice.ts implementing batch/single fetch, enrichment, caching, health checks and provider stats
- Export new stockdata module and classes from ts/stocks/index.ts
- Add comprehensive tests: test/test.stockdata.service.node.ts to cover setup, provider registration, fetching, caching, enrichment, health and error handling
- Update README with Unified Stock Data API examples, usage, and documentation reflecting new unified service
- Minor infra: add .claude/settings.local.json permissions for local tooling and web fetch domains
## 2025-11-01 - 3.1.0 - feat(fundamentals)
Add FundamentalsService and SEC EDGAR provider with caching, rate-limiting, tests, and docs updates
- Introduce FundamentalsService to manage fundamentals providers, caching, retry logic and provider statistics
- Add SecEdgarProvider to fetch SEC EDGAR company facts (CIK lookup, company facts parsing) with rate limiting and local caches
- Expose fundamentals interfaces and services from ts/stocks (exports updated)
- Add comprehensive tests for FundamentalsService and SecEdgarProvider (new test files)
- Update README with new Fundamentals module documentation, usage examples, and configuration guidance
- Implement caching and TTL handling for fundamentals data and provider-specific cache TTL support
- Add .claude/settings.local.json (local permissions) and various test improvements
## 2025-10-31 - 3.0.0 - BREAKING CHANGE(stocks)
Unify stock provider API to discriminated IStockDataRequest and add company name/fullname enrichment
- Replace legacy provider methods (fetchPrice/fetchPrices) with a single fetchData(request: IStockDataRequest) on IStockProvider — providers must be migrated to the new signature.
- Migrate StockPriceService to the unified getData(request: IStockDataRequest) API. Convenience helpers getPrice/getPrices now wrap getData.
- Add companyName and companyFullName fields to IStockPrice and populate them in provider mappings (Marketstack mapping updated; Yahoo provider updated to support the unified API).
- MarketstackProvider: added buildCompanyFullName helper and improved mapping to include company identification fields and full name formatting.
- YahooFinanceProvider: updated to implement fetchData and to route current/batch requests through the new unified request types; historical/intraday throw explicit errors.
- Updated tests to exercise the new unified API, company-name enrichment, caching behavior, and provider direct methods.
- Note: This is a breaking change for external providers and integrations that implemented the old fetchPrice/fetchPrices API. Bump major version.
## 2025-10-31 - 2.1.0 - feat(stocks)
Add unified stock data API (getData) with historical/OHLCV support, smart caching and provider enhancements
- Introduce discriminated union request types (IStockDataRequest) and a unified getData() method (replaces legacy getPrice/getPrices for new use cases)
- Add OHLCV fields (open, high, low, volume, adjusted) and metadata (dataType, fetchedAt) to IStockPrice
- Implement data-type aware smart caching with TTLs (historical = never expire, EOD = 24h, live = 30s, intraday matches interval)
- Extend StockPriceService: new getData(), data-specific cache keys, cache maxEntries increased (default 10000), and TTL-aware add/get cache logic
- Enhance Marketstack provider: unified fetchData(), historical date-range retrieval with pagination, exchange filtering, batch current fetch, OHLCV mapping, and intraday placeholder
- Update Yahoo provider to include dataType and fetchedAt (live data) and maintain legacy fetchPrice/fetchPrices compatibility
- Add/adjust tests to cover unified API, historical retrieval, OHLCV presence and smart caching behavior; test setup updated to require explicit OpenData directory paths
- Update README to document v2.1 changes, migration examples, and new stock provider capabilities
## 2025-10-31 - 2.0.0 - BREAKING CHANGE(OpenData)
Require explicit directory paths for OpenData (nogit/download/germanBusinessData); remove automatic .nogit creation; update HandelsRegister, JsonlDataProcessor, tests and README.
- Breaking: OpenData constructor now requires a config object with nogitDir, downloadDir and germanBusinessDataDir. The constructor will throw if these paths are not provided.
- Removed automatic creation/export of .nogit/download/germanBusinessData from ts/paths. OpenData.start now ensures the required directories exist.
- HandelsRegister API changed: constructor now accepts downloadDir and manages its own unique download folder; screenshot and download paths now use the configured downloadDir.
- JsonlDataProcessor now accepts a germanBusinessDataDir parameter and uses it when ensuring/storing data instead of relying on global paths.
- Updated tests to provide explicit path configuration (tests now set testNogitDir, testDownloadDir, testGermanBusinessDataDir and write outputs accordingly) and to use updated constructors and qenv usage.
- Documentation updated (README) to document the breaking change and show examples for required directory configuration when instantiating OpenData.
- Added .claude/settings.local.json for local permissions/config used in development/CI environments.
## 2025-10-11 - 1.7.0 - feat(stocks)
Add Marketstack provider (EOD) with tests, exports and documentation updates
- Add MarketstackProvider implementation (ts/stocks/providers/provider.marketstack.ts) providing EOD single and batch fetching, availability checks and mapping to IStockPrice.
- Export MarketstackProvider from ts/stocks/index.ts so it is available via the public API.
- Add comprehensive Marketstack tests (test/test.marketstack.node.ts) covering registration, health checks, single/batch fetches, caching, ticker/market validation, provider stats and sample output.
- Update README with Marketstack usage examples, configuration, API key instructions and provider/health documentation.
- Bump dev dependency @git.zone/tstest to ^2.4.2 in package.json.
- Add project helper/config files (.claude/settings.local.json, .serena/project.yml and .serena/.gitignore) to support CI/tooling.
## 2025-09-24 - 1.6.1 - fix(stocks)
Fix Yahoo provider request handling and bump dependency versions
- Refactored Yahoo Finance provider to use SmartRequest.create() builder and await response.json() for HTTP responses (replaces direct getJson usage).
- Improved batch and single-price fetching to use the SmartRequest API, keeping User-Agent header and timeouts.
- Added a compile-time type-check alias to ensure IStockPrice matches tsclass.finance.IStockPrice.
- Bumped development and runtime dependency versions (notable bumps include @git.zone/tsbuild, @git.zone/tstest, @push.rocks/qenv, @push.rocks/smartarchive, @push.rocks/smartdata, @push.rocks/smartfile, @push.rocks/smartlog, @push.rocks/smartpath, @push.rocks/smartrequest, @tsclass/tsclass).
- Added .claude/settings.local.json to grant local CI permissions for a few Bash commands.
## 2025-07-12 - 1.6.0 - feat(readme)
Revamp documentation and package description for enhanced clarity
- Restructured README to highlight real-time stock data and German business data, streamlining quick start and advanced examples
- Updated package.json description to better reflect library capabilities
- Added .claude/settings.local.json to define permissions for external tools
- Refined code examples in tests and documentation for improved clarity and consistency
## 2025-04-09 - 1.5.3 - fix(test)
Await file writes in Handelsregister tests to ensure all downloads complete before test end
- Replaced array.map with await Promise.all to properly await asynchronous file writes in test/test.handelsregister.ts
- Improved robustness of asynchronous operations in test suite
## 2025-04-09 - 1.5.2 - fix(readme)
Improve .env configuration code block formatting in documentation
- Wrap the .env variables block in triple backticks for clarity
- Ensure consistency in the Markdown styling of code snippets
## 2025-04-09 - 1.5.1 - fix(core)
No changes detected in project files or documentation. This commit is a placeholder to record that nothing was updated.
## 2025-04-09 - 1.5.0 - feat(documentation)
Enhance project metadata and documentation with comprehensive usage examples, updated descriptions, and improved keywords.
- Updated npmextra.json and package.json to refine the project description and keyword list.
- Expanded readme.md with detailed sections on environment setup, CRUD operations, bulk JSONL processing, and advanced Handelsregister integrations.
- Included advanced workflow examples and error handling strategies in the documentation.
- Adjusted test cases (e.g. in test/test.handelsregister.ts) to reflect changes in company name usage.
## 2025-04-08 - 1.4.6 - fix(tests & jsonl)
Improve test structure and refine JSONL parsing for incomplete data
- Refactored test files to remove redundant get-specific-company tests in test.ts and added missing tests in test.handelsregister.ts
- Updated JSONL data processor to conditionally parse remaining data when available
## 2025-04-05 - 1.4.5 - fix(metadata)
Update repository, bugs, and homepage URLs to code.foss.global
- Repository URL updated from gitlab.com to code.foss.global
- Bugs URL updated from gitlab.com to code.foss.global
- Homepage URL updated to code.foss.global
## 2025-04-05 - 1.4.4 - fix(dependencies & tests)
Update dependency versions and refine test search query
- Bumped versions for several dependencies in package.json, including @git.zone/tsbuild, @git.zone/tsbundle, @git.zone/tstest, @push.rocks/tapbundle, @push.rocks/smartdata, @push.rocks/smartfile, @push.rocks/smartpromise, @push.rocks/smartrequest, and @tsclass/tsclass
- Updated test file to replace the search query 'Volkswagen' with 'LADR'
- Re-enabled the build initial data test by removing tap.skip
## 2025-01-07 - 1.4.3 - fix(test)
Corrected index value in test for fetching specific company data

7209
deno.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -5,25 +5,26 @@
"githost": "gitlab.com",
"gitscope": "fin.cx",
"gitrepo": "opendata",
"description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
"description": "A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.",
"npmPackagename": "@fin.cx/opendata",
"license": "MIT",
"projectDomain": "fin.cx",
"keywords": [
"TypeScript",
"open data",
"business data",
"German companies",
"data management",
"business registry",
"npm package",
"business data",
"MongoDB",
"JSONL",
"bulk processing",
"data management",
"automation",
"data integration",
"database",
"data processing",
"data retrieval",
"data update"
"browser automation",
"Handelsregister",
"web scraping",
"file processing",
"business registry",
"data retrieval"
]
}
},

View File

@@ -1,51 +1,51 @@
{
"name": "@fin.cx/opendata",
"version": "1.4.3",
"version": "3.5.0",
"private": false,
"description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
"description": "A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"author": "Task Venture Capital GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/ --web)",
"test": "(tstest test/ --verbose)",
"build": "(tsbuild --web --allowimplicitany)",
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.2.0",
"@git.zone/tsbundle": "^2.1.0",
"@git.zone/tsrun": "^1.3.3",
"@git.zone/tstest": "^1.0.90",
"@push.rocks/tapbundle": "^5.5.4",
"@types/node": "^22.10.5"
"@git.zone/tsbuild": "^2.6.8",
"@git.zone/tsbundle": "^2.5.1",
"@git.zone/tsrun": "^1.6.2",
"@git.zone/tstest": "^2.7.0",
"@types/node": "^22.14.0"
},
"dependencies": {
"@push.rocks/lik": "^6.1.0",
"@push.rocks/qenv": "^6.1.0",
"@push.rocks/smartarchive": "^4.0.39",
"@push.rocks/lik": "^6.2.2",
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/smartarchive": "^4.2.2",
"@push.rocks/smartarray": "^1.1.0",
"@push.rocks/smartbrowser": "^2.0.8",
"@push.rocks/smartdata": "^5.2.10",
"@push.rocks/smartdata": "^5.16.4",
"@push.rocks/smartdelay": "^3.0.5",
"@push.rocks/smartfile": "^11.1.5",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.1.0",
"@push.rocks/smartrequest": "^2.0.23",
"@push.rocks/smartfile": "^11.2.7",
"@push.rocks/smartlog": "^3.1.10",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^4.3.4",
"@push.rocks/smartstream": "^3.2.5",
"@push.rocks/smartunique": "^3.0.9",
"@push.rocks/smartxml": "^1.1.1",
"@tsclass/tsclass": "^4.2.0"
"@tsclass/tsclass": "^9.3.0"
},
"repository": {
"type": "git",
"url": "git+https://gitlab.com/fin.cx/opendata.git"
"url": "https://code.foss.global/fin.cx/opendata.git"
},
"bugs": {
"url": "https://gitlab.com/fin.cx/opendata/issues"
"url": "https://code.foss.global/fin.cx/opendata/issues"
},
"homepage": "https://gitlab.com/fin.cx/opendata#readme",
"homepage": "https://code.foss.global/fin.cx/opendata#readme",
"browserslist": [
"last 1 chrome versions"
],
@@ -64,17 +64,19 @@
"keywords": [
"TypeScript",
"open data",
"business data",
"German companies",
"data management",
"business registry",
"npm package",
"business data",
"MongoDB",
"JSONL",
"bulk processing",
"data management",
"automation",
"data integration",
"database",
"data processing",
"data retrieval",
"data update"
]
"browser automation",
"Handelsregister",
"web scraping",
"file processing",
"business registry",
"data retrieval"
],
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
}

7847
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,89 @@
# OpenData Project Hints
## Stocks Module
### Overview
The stocks module provides real-time stock price data and cryptocurrency prices through various provider implementations. Currently supports Yahoo Finance, Marketstack, and CoinGecko with an extensible architecture for additional providers.
### Architecture
- **Provider Pattern**: Each stock data source implements the `IStockProvider` interface
- **Service Registry**: `StockPriceService` manages providers with priority-based selection
- **Caching**: Built-in cache with configurable TTL to reduce API calls
- **Fallback Logic**: Automatic failover between providers if one fails
### Yahoo Finance Provider Notes
- Uses public API endpoints (no authentication required)
- Two main endpoints:
- `/v8/finance/chart/{ticker}` - Single ticker with full data
- `/v8/finance/spark?symbols={tickers}` - Multiple tickers with basic data
- Response data is in `response.body` when using smartrequest
- Requires User-Agent header to avoid rate limiting
### Usage Example
```typescript
import { StockPriceService, YahooFinanceProvider } from '@fin.cx/opendata';
const stockService = new StockPriceService({ ttl: 60000 });
const yahooProvider = new YahooFinanceProvider();
stockService.register(yahooProvider);
const price = await stockService.getPrice({ ticker: 'AAPL' });
console.log(`${price.ticker}: $${price.price}`);
```
### CoinGecko Provider Notes
- Cryptocurrency price provider supporting 13M+ tokens
- Three main endpoints:
- `/simple/price` - Current prices with market data (batch supported)
- `/coins/{id}/market_chart` - Historical and intraday prices with OHLCV
- `/coins/list` - Complete coin list for ticker-to-ID mapping
- **Rate Limiting**:
- Free tier: 5-15 calls/minute (no registration)
- Demo plan: 30 calls/minute, 10,000/month (free with registration)
- Custom rate limiter tracks requests per minute
- **Ticker Resolution**:
- Accepts both ticker symbols (BTC, ETH) and CoinGecko IDs (bitcoin, ethereum)
- Lazy-loads coin list on first ticker resolution
- Caches coin mappings for 24 hours
- IDs with hyphens (wrapped-bitcoin) assumed to be CoinGecko IDs
- **24/7 Markets**: Crypto markets always return `marketState: 'REGULAR'`
- **Optional API Key**: Pass key to constructor for higher rate limits
- Demo plan: `x-cg-demo-api-key` header
- Paid plans: `x-cg-pro-api-key` header
- **Data Granularity**:
- Historical: Daily data for date ranges
- Intraday: Hourly data only (1-90 days based on `days` param)
- Current: Real-time prices with 24h change and volume
### Usage Example (Crypto)
```typescript
import { StockPriceService, CoinGeckoProvider } from '@fin.cx/opendata';
const stockService = new StockPriceService({ ttl: 30000 });
const coingeckoProvider = new CoinGeckoProvider(); // or new CoinGeckoProvider('api-key')
stockService.register(coingeckoProvider);
// Using ticker symbol
const btc = await stockService.getPrice({ ticker: 'BTC' });
console.log(`${btc.ticker}: $${btc.price}`);
// Using CoinGecko ID
const ethereum = await stockService.getPrice({ ticker: 'ethereum' });
// Batch fetch
const cryptos = await stockService.getPrices({ tickers: ['BTC', 'ETH', 'USDT'] });
```
### Testing
- Tests use real API calls (be mindful of rate limits)
- Mock invalid ticker 'INVALID_TICKER_XYZ' for error testing
- Clear cache between tests to ensure fresh data
- The spark endpoint may return fewer results than requested
- CoinGecko tests may take longer due to rate limiting (wait between requests)
### Future Providers
To add a new provider:
1. Create `ts/stocks/providers/provider.{name}.ts`
2. Implement the `IStockProvider` interface
3. Register with `StockPriceService`
4. No changes needed to existing code

875
readme.md
View File

@@ -1,220 +1,785 @@
# @fin.cx/opendata
A TypeScript-based library for accessing and managing open business data, specifically for German companies.
🚀 **Complete financial intelligence toolkit for TypeScript**
## Install
Access real-time stock prices, fundamental financial data, and comprehensive German company information - all through a single, unified API.
To install the `@fin.cx/opendata` package, you can use npm or yarn as your package manager. Here's how you can do it:
Using npm:
## Installation
```bash
npm install @fin.cx/opendata
# or
pnpm add @fin.cx/opendata
```
Using yarn:
## Quick Start
```bash
yarn add @fin.cx/opendata
### ✨ Unified Stock Data API (Recommended)
Get complete stock data with automatic enrichment - the elegant way:
```typescript
import { StockDataService, YahooFinanceProvider, SecEdgarProvider } from '@fin.cx/opendata';
// Initialize unified service
const stockData = new StockDataService();
// Register providers
stockData.registerPriceProvider(new YahooFinanceProvider());
stockData.registerFundamentalsProvider(new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com'
}));
// ✨ Get complete stock data with ONE method call
const apple = await stockData.getStockData('AAPL');
console.log({
company: apple.fundamentals.companyName, // "Apple Inc."
price: apple.price.price, // $270.37
marketCap: apple.fundamentals.marketCap, // $4.13T (auto-calculated!)
peRatio: apple.fundamentals.priceToEarnings, // 28.42 (auto-calculated!)
pbRatio: apple.fundamentals.priceToBook, // 45.12 (auto-calculated!)
eps: apple.fundamentals.earningsPerShareDiluted, // $6.13
revenue: apple.fundamentals.revenue // $385.6B
});
// Batch fetch with automatic enrichment
const stocks = await stockData.getBatchStockData(['AAPL', 'MSFT', 'GOOGL']);
stocks.forEach(stock => {
console.log(`${stock.ticker}: $${stock.price.price.toFixed(2)}, ` +
`P/E ${stock.fundamentals?.priceToEarnings?.toFixed(2)}`);
});
// Output:
// AAPL: $270.37, P/E 28.42
// MSFT: $425.50, P/E 34.21
// GOOGL: $142.15, P/E 25.63
```
## Usage
**Why use the unified API?**
-**Single service** for both prices and fundamentals
-**Automatic enrichment** - Market cap, P/E, P/B calculated automatically
-**One method call** - No manual `enrichWithPrice()` calls
-**Simplified code** - Less boilerplate, more readable
-**Type-safe** - Full TypeScript support
The `@fin.cx/opendata` package provides a comprehensive set of functionalities for handling open business data, focusing on German business data. Let's explore the detailed capabilities of this library through extensive examples and instructions.
### 📈 Price-Only Data (Alternative)
### Setting Up the Environment
If you only need prices without fundamentals:
First, make sure you've set up the necessary environment variables for MongoDB. You will need the following environment variables:
- `MONGODB_URL`: The URL for your MongoDB instance.
- `MONGODB_NAME`: The name of the database to use.
- `MONGODB_USER`: A valid username for accessing the database.
- `MONGODB_PASS`: The password associated with the MongoDB user.
```typescript
import { StockDataService, YahooFinanceProvider } from '@fin.cx/opendata';
These variables can be configured in a `.env` file or managed through a specific service used for secure environment variables handling.
const stockData = new StockDataService();
stockData.registerPriceProvider(new YahooFinanceProvider());
### Importing and Initializing the Library
// Get just the price
const price = await stockData.getPrice('AAPL');
console.log(`${price.ticker}: $${price.price}`);
To start working with the library, import the necessary classes and initialize the `OpenData` class.
// Or use StockPriceService directly for more control
import { StockPriceService } from '@fin.cx/opendata';
const stockService = new StockPriceService({ ttl: 60000 });
stockService.register(new YahooFinanceProvider());
const apple = await stockService.getData({ type: 'current', ticker: 'AAPL' });
console.log(`${apple.companyFullName}: $${apple.price}`);
```
### 🪙 Cryptocurrency Prices
Get real-time crypto prices with the CoinGecko provider:
```typescript
import { StockPriceService, CoinGeckoProvider } from '@fin.cx/opendata';
const stockService = new StockPriceService({ ttl: 30000 });
// Optional: Pass API key for higher rate limits
// const provider = new CoinGeckoProvider('your-api-key');
const provider = new CoinGeckoProvider();
stockService.register(provider);
// Fetch single crypto price (using ticker symbol)
const btc = await stockService.getPrice({ ticker: 'BTC' });
console.log(`${btc.ticker}: $${btc.price.toLocaleString()}`);
console.log(`24h Change: ${btc.changePercent.toFixed(2)}%`);
console.log(`24h Volume: $${btc.volume?.toLocaleString()}`);
// Or use CoinGecko ID directly
const ethereum = await stockService.getPrice({ ticker: 'ethereum' });
// Batch fetch multiple cryptos
const cryptos = await stockService.getPrices({
tickers: ['BTC', 'ETH', 'USDT', 'BNB', 'SOL']
});
cryptos.forEach(crypto => {
console.log(`${crypto.ticker}: $${crypto.price.toFixed(2)} (${crypto.changePercent >= 0 ? '+' : ''}${crypto.changePercent.toFixed(2)}%)`);
});
// Historical crypto prices
const history = await stockService.getData({
type: 'historical',
ticker: 'BTC',
from: new Date('2025-01-01'),
to: new Date('2025-01-31')
});
// Intraday crypto prices (hourly)
const intraday = await stockService.getData({
type: 'intraday',
ticker: 'ETH',
interval: '1hour',
limit: 24 // Last 24 hours
});
```
### 💰 Fundamentals-Only Data (Alternative)
If you only need fundamentals without prices:
```typescript
import { StockDataService, SecEdgarProvider } from '@fin.cx/opendata';
const stockData = new StockDataService();
stockData.registerFundamentalsProvider(new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com'
}));
// Get just fundamentals
const fundamentals = await stockData.getFundamentals('AAPL');
console.log({
company: fundamentals.companyName,
eps: fundamentals.earningsPerShareDiluted,
revenue: fundamentals.revenue,
sharesOutstanding: fundamentals.sharesOutstanding
});
// Or use FundamentalsService directly
import { FundamentalsService } from '@fin.cx/opendata';
const fundamentalsService = new FundamentalsService();
fundamentalsService.register(new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com'
}));
const data = await fundamentalsService.getFundamentals('AAPL');
```
### 🏢 German Business Intelligence
Access comprehensive German company data:
```typescript
import { OpenData } from '@fin.cx/opendata';
import * as path from 'path';
const initializeOpenData = async () => {
const openData = new OpenData();
try {
// Configure directory paths
const openData = new OpenData({
nogitDir: path.join(process.cwd(), '.nogit'),
downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
});
await openData.start();
console.log('OpenData instance has started successfully.');
// Example usage:
await createAndManageBusinessRecords(openData);
// Search for companies
const results = await openData.handelsregister.searchCompany("Siemens AG");
} catch (error) {
console.error('Error starting OpenData:', error);
} finally {
await openData.stop();
console.log('OpenData instance has stopped.');
// Get detailed information with documents
const details = await openData.handelsregister.getSpecificCompany({
court: "Munich",
type: "HRB",
number: "6684"
});
```
## Features
### 📊 Stock & Crypto Market Module
- **Real-Time Prices** - Live and EOD prices from Yahoo Finance, Marketstack, and CoinGecko
- **Cryptocurrency Support** - 13M+ crypto tokens with 24/7 market data via CoinGecko
- **Company Names** - Automatic company name extraction (e.g., "Apple Inc (NASDAQ:AAPL)")
- **Historical Data** - Up to 15 years of daily EOD prices with pagination
- **OHLCV Data** - Open, High, Low, Close, Volume for technical analysis
- **Exchange Filtering** - Query specific exchanges via MIC codes (XNAS, XLON, XNYS)
- **Smart Caching** - Data-type aware TTL (historical cached forever, EOD 24h, live 30s)
- **Batch Operations** - Fetch 100+ symbols in one request
- **Type-Safe API** - Full TypeScript support with discriminated unions
- **Multi-Provider** - Automatic fallback between providers
### 💰 Fundamental Data Module
- **SEC EDGAR Integration** - FREE fundamental data directly from SEC filings
- **Comprehensive Metrics** - EPS, Revenue, Assets, Liabilities, Cash Flow, and more
- **All US Public Companies** - Complete coverage of SEC-registered companies
- **Historical Filings** - Data back to ~2009
- **CIK Lookup** - Automatic ticker-to-CIK mapping with smart caching
- **Calculated Ratios** - Market Cap, P/E, P/B ratios when combined with prices
- **No API Key Required** - Direct access to SEC's public API
- **Rate Limit Management** - Built-in 10 req/sec rate limiting
### 🇩🇪 German Business Intelligence
- **MongoDB Integration** - Scalable data storage for millions of records
- **Bulk JSONL Import** - Process multi-GB datasets efficiently
- **Handelsregister Automation** - Automated document retrieval
- **CRUD Operations** - Full database management with validation
- **Streaming Processing** - Handle large datasets without memory issues
## Advanced Examples
### Combined Market Analysis (Unified API)
Analyze multiple companies with automatic enrichment:
```typescript
import { StockDataService, YahooFinanceProvider, SecEdgarProvider } from '@fin.cx/opendata';
// Setup unified service
const stockData = new StockDataService();
stockData.registerPriceProvider(new YahooFinanceProvider());
stockData.registerFundamentalsProvider(new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com'
}));
// Analyze multiple companies with ONE call
const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN'];
const stocks = await stockData.getBatchStockData(tickers);
// All metrics are automatically calculated!
stocks.forEach(stock => {
if (stock.fundamentals) {
console.log(`\n${stock.fundamentals.companyName} (${stock.ticker})`);
console.log(` Price: $${stock.price.price.toFixed(2)}`);
console.log(` Market Cap: $${(stock.fundamentals.marketCap! / 1e9).toFixed(2)}B`);
console.log(` P/E Ratio: ${stock.fundamentals.priceToEarnings!.toFixed(2)}`);
console.log(` Revenue: $${(stock.fundamentals.revenue! / 1e9).toFixed(2)}B`);
console.log(` EPS: $${stock.fundamentals.earningsPerShareDiluted!.toFixed(2)}`);
}
};
});
initializeOpenData();
```
// Or analyze one-by-one with automatic enrichment
for (const ticker of tickers) {
const stock = await stockData.getStockData(ticker);
### Managing Business Records
The `BusinessRecord` class represents a company's data. Let's explore how you can create, retrieve, update, and manage these records.
#### Creating a New BusinessRecord
Creating a new business record involves instantiating the `BusinessRecord` class and setting the relevant properties.
```typescript
import { BusinessRecord } from '@fin.cx/opendata';
const createBusinessRecord = async (openData: OpenData) => {
const businessRecord = new openData.CBusinessRecord();
businessRecord.data = {
name: "Tech Innovations GmbH",
address: "Tech Park 42",
postalCode: "80333",
city: "Munich",
country: "Germany",
phone: "+49 89 123456",
email: "info@techinnovations.de",
website: "https://techinnovations.de",
businessType: "GmbH",
registrationNumber: "HRB 654321",
registrationCourt: "Munich",
legalForm: "GmbH",
managingDirectors: ["Alice Müller", "Bob Schmidt"],
foundingDate: new Date("2020-01-01").toISOString(),
capital: "100,000 EUR",
purpose: "Developing innovative tech solutions",
lastUpdate: new Date().toISOString()
};
await businessRecord.save();
console.log('BusinessRecord saved:', businessRecord);
};
```
#### Retrieving Business Records
You can retrieve existing business records by querying the database using various data fields.
```typescript
const retrieveBusinessRecords = async (openData: OpenData) => {
const records = await openData.db
.collection<BusinessRecord>('businessrecords')
.find({ city: "Munich" })
.toArray();
console.log('Found Business Records:', records);
};
```
#### Updating Existing Records
To update an existing `BusinessRecord`, you retrieve the record, modify its data, and save it again.
```typescript
const updateBusinessRecord = async (openData: OpenData, recordId: string) => {
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
if (businessRecord) {
businessRecord.data.phone = "+49 89 987654";
businessRecord.data.lastUpdate = new Date().toISOString();
await businessRecord.save();
console.log('BusinessRecord updated:', businessRecord);
} else {
console.log('BusinessRecord not found for id:', recordId);
// Everything is already enriched - no manual calculations needed!
console.log(`${ticker}: P/E ${stock.fundamentals?.priceToEarnings?.toFixed(2)}`);
}
};
```
#### Deleting a Business Record
### Fundamental Data Screening
You can delete a business record using its unique identifier.
Screen stocks by financial metrics:
```typescript
const deleteBusinessRecord = async (openData: OpenData, recordId: string) => {
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
if (businessRecord) {
await businessRecord.delete();
console.log(`BusinessRecord with id ${recordId} deleted successfully.`);
} else {
console.log('No record found for the provided id:', recordId);
// Fetch data for multiple tickers
const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'META', 'NVDA', 'TSLA'];
const stocks = await stockData.getBatchStockData(tickers);
// Filter by criteria (all metrics auto-calculated!)
const valueStocks = stocks.filter(stock => {
const f = stock.fundamentals;
return f &&
f.priceToEarnings! < 30 && // P/E under 30
f.priceToBook! < 10 && // P/B under 10
f.revenue! > 100_000_000_000; // Revenue > $100B
});
console.log('\n💎 Value Stocks:');
valueStocks.forEach(stock => {
console.log(`${stock.ticker}: P/E ${stock.fundamentals!.priceToEarnings!.toFixed(2)}, ` +
`P/B ${stock.fundamentals!.priceToBook!.toFixed(2)}`);
});
```
### Historical Data Analysis
Fetch and analyze historical price trends:
```typescript
// Get 1 year of historical data
const history = await stockService.getData({
type: 'historical',
ticker: 'AAPL',
from: new Date('2024-01-01'),
to: new Date('2024-12-31'),
sort: 'DESC' // Newest first
});
// Calculate statistics
const prices = history.map(p => p.price);
const high52Week = Math.max(...prices);
const low52Week = Math.min(...prices);
const avgPrice = prices.reduce((a, b) => a + b) / prices.length;
console.log(`52-Week Analysis for AAPL:`);
console.log(` High: $${high52Week.toFixed(2)}`);
console.log(` Low: $${low52Week.toFixed(2)}`);
console.log(` Average: $${avgPrice.toFixed(2)}`);
console.log(` Days: ${history.length}`);
// Calculate Simple Moving Average
const calculateSMA = (data: IStockPrice[], period: number) => {
const sma: number[] = [];
for (let i = period - 1; i < data.length; i++) {
const sum = data.slice(i - period + 1, i + 1)
.reduce((acc, p) => acc + p.price, 0);
sma.push(sum / period);
}
return sma;
};
const sma20 = calculateSMA(history, 20);
const sma50 = calculateSMA(history, 50);
console.log(`\nMoving Averages:`);
console.log(` 20-day SMA: $${sma20[0].toFixed(2)}`);
console.log(` 50-day SMA: $${sma50[0].toFixed(2)}`);
```
### Updating German Business Data
### OHLCV Technical Analysis
The package includes functionalities to keep your business data up-to-date by downloading from official German open data repositories.
Use OHLCV data for technical indicators:
```typescript
const updateGermanBusinessData = async (openData: OpenData) => {
const history = await stockService.getData({
type: 'historical',
ticker: 'TSLA',
from: new Date('2024-11-01'),
to: new Date('2024-11-30')
});
// Calculate daily trading range
for (const day of history) {
const range = day.high! - day.low!;
const rangePercent = (range / day.low!) * 100;
console.log(`${day.timestamp.toISOString().split('T')[0]}:`);
console.log(` Open: $${day.open}`);
console.log(` High: $${day.high}`);
console.log(` Low: $${day.low}`);
console.log(` Close: $${day.price}`);
console.log(` Volume: ${day.volume?.toLocaleString()}`);
console.log(` Range: $${range.toFixed(2)} (${rangePercent.toFixed(2)}%)`);
}
```
### Fundamental Data Screening
Screen stocks based on fundamental metrics:
```typescript
const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA', 'META', 'NVDA'];
// Fetch fundamentals for all tickers
const allFundamentals = await fundamentalsService.getBatchFundamentals(tickers);
// Get current prices
const prices = await stockService.getData({
type: 'batch',
tickers: tickers
});
// Create price map
const priceMap = new Map(prices.map(p => [p.ticker, p.price]));
// Enrich with prices
const enriched = await fundamentalsService.enrichBatchWithPrices(
allFundamentals,
priceMap
);
// Screen for value stocks (P/E < 25, P/B < 5)
const valueStocks = enriched.filter(f =>
f.priceToEarnings && f.priceToEarnings < 25 &&
f.priceToBook && f.priceToBook < 5
);
console.log('Value Stocks:');
valueStocks.forEach(stock => {
console.log(`\n${stock.companyName} (${stock.ticker})`);
console.log(` P/E Ratio: ${stock.priceToEarnings!.toFixed(2)}`);
console.log(` P/B Ratio: ${stock.priceToBook!.toFixed(2)}`);
console.log(` Market Cap: $${(stock.marketCap! / 1e9).toFixed(2)}B`);
});
```
### Market Dashboard
Create a comprehensive market overview:
```typescript
const indicators = [
{ ticker: 'AAPL', name: 'Apple' },
{ ticker: 'MSFT', name: 'Microsoft' },
{ ticker: 'GOOGL', name: 'Alphabet' },
{ ticker: 'AMZN', name: 'Amazon' },
{ ticker: 'TSLA', name: 'Tesla' }
];
const prices = await stockService.getData({
type: 'batch',
tickers: indicators.map(i => i.ticker)
});
// Display with color-coded changes
prices.forEach(price => {
const indicator = indicators.find(i => i.ticker === price.ticker);
const arrow = price.change >= 0 ? '↑' : '↓';
const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
console.log(
`${price.companyName!.padEnd(25)} $${price.price.toFixed(2).padStart(8)} ` +
`${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m`
);
});
```
### Exchange-Specific Trading
Compare prices across different exchanges:
```typescript
// Vodafone trades on both London and NYSE
const exchanges = [
{ mic: 'XLON', name: 'London Stock Exchange' },
{ mic: 'XNYS', name: 'New York Stock Exchange' }
];
for (const exchange of exchanges) {
try {
await openData.germanBusinesses.update();
console.log('German business data has been updated successfully.');
const price = await stockService.getData({
type: 'current',
ticker: 'VOD',
exchange: exchange.mic
});
console.log(`${exchange.name}:`);
console.log(` Price: ${price.price} ${price.currency}`);
console.log(` Volume: ${price.volume?.toLocaleString()}`);
console.log(` Exchange: ${price.exchangeName}`);
} catch (error) {
console.error('Error updating German business data:', error);
console.log(`${exchange.name}: Not available`);
}
}
};
```
This function fetches the latest open data regarding German companies, processes it, and updates your local database.
## Configuration
### Detailed Class Insights
#### OpenData Class
The `OpenData` class serves as the core of the library, initializing necessary components and controlling data flows:
- **db**: Represents the connection to your MongoDB database.
- **germanBusinesses**: An instance handling specific operations related to German business data updates.
### Stock Service Options
```typescript
class OpenData {
db: plugins.smartdata.SmartdataDb;
germanBusinesses: GermanBusinessData;
const stockService = new StockPriceService({
ttl: 60000, // Default cache TTL in ms
maxEntries: 10000 // Max cached entries
});
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
// Marketstack - EOD data (requires API key)
stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
enabled: true,
priority: 100,
timeout: 10000,
retryAttempts: 3,
retryDelay: 1000
});
public async start() {
// Database initialization logic
}
public async stop() {
// Cleanup logic
}
}
// Yahoo Finance - Real-time data (no API key)
stockService.register(new YahooFinanceProvider(), {
enabled: true,
priority: 50
});
```
#### GermanBusinessData Class
This class deals specifically with German company data — fetching, processing, and updating local databases consistently with official German data sources.
### Fundamentals Service Options
```typescript
class GermanBusinessData {
public async start() {
await this.update();
}
const fundamentalsService = new FundamentalsService({
ttl: 90 * 24 * 60 * 60 * 1000, // 90 days (quarterly refresh)
maxEntries: 10000
});
public async update() {
// Logic for updating business data using import streams and parsing JSON lines.
}
// SEC EDGAR provider (FREE - no API key!)
fundamentalsService.register(new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com',
cikCacheTTL: 30 * 24 * 60 * 60 * 1000, // 30 days
fundamentalsCacheTTL: 90 * 24 * 60 * 60 * 1000, // 90 days
timeout: 30000
}));
```
### Directory Configuration (German Business Data)
All directory paths are mandatory when using German business data features:
```typescript
import { OpenData } from '@fin.cx/opendata';
import * as path from 'path';
// Development
const openData = new OpenData({
nogitDir: path.join(process.cwd(), '.nogit'),
downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
});
// Production
const openDataProd = new OpenData({
nogitDir: '/var/lib/myapp/data',
downloadDir: '/var/lib/myapp/data/downloads',
germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata'
});
```
### Environment Variables
Set environment variables for API keys and database:
```env
# Marketstack API (for EOD stock data)
MARKETSTACK_COM_TOKEN=your_api_key_here
# MongoDB (for German business data)
MONGODB_URL=mongodb://localhost:27017
MONGODB_NAME=opendata
MONGODB_USER=myuser
MONGODB_PASS=mypass
```
## API Reference
### Stock Price Interfaces
```typescript
interface IStockPrice {
ticker: string;
price: number;
currency: string;
change: number;
changePercent: number;
previousClose: number;
timestamp: Date;
provider: string;
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
exchange?: string;
exchangeName?: string;
// OHLCV data
volume?: number;
open?: number;
high?: number;
low?: number;
adjusted?: boolean;
dataType: 'eod' | 'intraday' | 'live';
fetchedAt: Date;
// Company identification
companyName?: string; // "Apple Inc"
companyFullName?: string; // "Apple Inc (NASDAQ:AAPL)"
}
```
### Ensuring Data Accuracy and Integrity
### Fundamental Data Interfaces
When working with business data, ensuring integrity and accuracy is crucial. Each record should be checked for validity before being saved or updated, minimizing inconsistencies. Moreover, robust error handling is essential in every step, from data retrieval to database operations, particularly when dealing with external data sources.
```typescript
interface IStockFundamentals {
ticker: string;
cik: string;
companyName: string;
provider: string;
timestamp: Date;
fetchedAt: Date;
The `@fin.cx/opendata` module provides an extensive toolset for accessing and managing business data, particularly for companies based in Germany. Its functionalities include creating, updating, retrieving, and deleting business records, as well as keeping them current with the latest open data releases. This makes it an invaluable asset for developers aiming to integrate open data seamlessly into their systems, ensuring robust data management capabilities within their applications.
// Per-share metrics
earningsPerShareBasic?: number;
earningsPerShareDiluted?: number;
sharesOutstanding?: number;
Happy exploring and integrating open data into your projects!
// Income statement (annual USD)
revenue?: number;
netIncome?: number;
operatingIncome?: number;
grossProfit?: number;
// Balance sheet (annual USD)
assets?: number;
liabilities?: number;
stockholdersEquity?: number;
cash?: number;
propertyPlantEquipment?: number;
// Calculated metrics (requires price)
marketCap?: number; // price × sharesOutstanding
priceToEarnings?: number; // price / EPS
priceToBook?: number; // marketCap / stockholdersEquity
// Metadata
fiscalYear?: string;
fiscalQuarter?: string;
filingDate?: Date;
form?: '10-K' | '10-Q' | string;
}
```
### Key Methods
**StockPriceService**
- `getData(request)` - Unified method for all stock data (current, historical, batch)
- `getPrice(request)` - Convenience method for single current price
- `getPrices(request)` - Convenience method for batch current prices
- `register(provider, config)` - Add data provider with priority and retry config
- `checkProvidersHealth()` - Test all providers and return health status
- `getProviderStats()` - Get success/error statistics for each provider
- `clearCache()` - Clear price cache
- `setCacheTTL(ttl)` - Update cache TTL dynamically
**FundamentalsService**
- `getFundamentals(ticker)` - Get fundamental data for single ticker
- `getBatchFundamentals(tickers)` - Get fundamentals for multiple tickers
- `enrichWithPrice(fundamentals, price)` - Calculate market cap, P/E, P/B ratios
- `enrichBatchWithPrices(fundamentals, priceMap)` - Batch enrich with prices
- `register(provider, config)` - Add fundamentals provider
- `checkProvidersHealth()` - Test all providers
- `getProviderStats()` - Get success/error statistics
- `clearCache()` - Clear fundamentals cache
**SecEdgarProvider**
- ✅ FREE - No API key required
- ✅ All US public companies
- ✅ Comprehensive US GAAP financial metrics
- ✅ Historical data back to ~2009
- ✅ Direct access to SEC filings (10-K, 10-Q)
- ✅ Smart caching (CIK: 30 days, Fundamentals: 90 days)
- ✅ Rate limiting (10 requests/second)
- Requires User-Agent header in format: "Company Name Email"
**MarketstackProvider**
- ✅ End-of-Day (EOD) stock prices
- ✅ 500,000+ tickers across 72+ exchanges worldwide
- ✅ Historical data with pagination
- ✅ Batch fetching support
- ✅ OHLCV data (Open, High, Low, Close, Volume)
- ✅ Company names included automatically
- ⚠️ Requires API key (free tier: 100 requests/month)
**YahooFinanceProvider**
- ✅ Real-time stock prices
- ✅ No API key required
- ✅ Global coverage
- ✅ Company names included
- ⚠️ Rate limits may apply
**CoinGeckoProvider**
- ✅ Cryptocurrency prices (Bitcoin, Ethereum, 13M+ tokens)
- ✅ Current, historical, and intraday data
- ✅ 24/7 market data (crypto never closes)
- ✅ OHLCV data with market cap and volume
- ✅ Supports both ticker symbols (BTC, ETH) and CoinGecko IDs (bitcoin, ethereum)
- ✅ 240+ networks, 1600+ exchanges
- Optional API key (free tier: 30 requests/min, 10K/month)
**OpenData**
- `start()` - Initialize MongoDB connection
- `buildInitialDb()` - Import bulk data
- `CBusinessRecord` - Business record class
- `handelsregister` - German registry automation
## Provider Architecture
Add custom data providers easily:
```typescript
class MyCustomProvider implements IStockProvider {
name = 'My Provider';
priority = 50;
requiresAuth = true;
rateLimit = { requestsPerMinute: 60 };
async fetchData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
// Implement unified data fetching
switch (request.type) {
case 'current':
return this.fetchCurrentPrice(request);
case 'batch':
return this.fetchBatchPrices(request);
case 'historical':
return this.fetchHistoricalPrices(request);
default:
throw new Error(`Unsupported request type`);
}
}
async isAvailable(): Promise<boolean> {
// Health check
return true;
}
supportsMarket(market: string): boolean {
return ['US', 'UK', 'DE'].includes(market);
}
supportsTicker(ticker: string): boolean {
return /^[A-Z]{1,5}$/.test(ticker);
}
}
stockService.register(new MyCustomProvider());
```
## Performance
- **Batch Fetching**: Get 100+ prices in one API request
- **Smart Caching**: Data-type aware TTL (historical cached forever, EOD 24h, live 30s)
- **Rate Limit Management**: Automatic retry logic for API limits
- **Concurrent Processing**: Handle 1000+ records/second
- **Streaming**: Process GB-sized datasets without memory issues
- **Provider Fallback**: Automatic failover between data sources
## Testing
Run the comprehensive test suite:
```bash
pnpm test
```
Test specific modules:
```bash
# Stock price providers
pnpm tstest test/test.marketstack.node.ts --verbose
pnpm tstest test/test.stocks.ts --verbose
# Fundamental data
pnpm tstest test/test.secedgar.provider.node.ts --verbose
pnpm tstest test/test.fundamentals.service.node.ts --verbose
# German business data
pnpm tstest test/test.handelsregister.ts --verbose
```
## Getting API Keys
### Marketstack (EOD Stock Data)
1. Visit [marketstack.com](https://marketstack.com)
2. Sign up for a free account (100 requests/month)
3. Get your API key from the dashboard
4. Set environment variable: `MARKETSTACK_COM_TOKEN=your_key_here`
### SEC EDGAR (Fundamental Data)
**No API key required!** SEC EDGAR is completely free and public. Just provide your company name and email in the User-Agent:
```typescript
new SecEdgarProvider({
userAgent: 'YourCompany youremail@example.com'
});
```
## License and Legal Information

193
readme.plan.md Normal file
View File

@@ -0,0 +1,193 @@
# Stock Prices Module Implementation Plan
Command to reread guidelines: Read /home/philkunz/.claude/CLAUDE.md
## Overview
Implementation of a stocks module for fetching current stock prices using various APIs. The architecture will support multiple providers, but we'll start with implementing only Yahoo Finance API. The design will make it easy to add additional providers (Alpha Vantage, IEX Cloud, etc.) in the future without changing the core architecture.
## Phase 1: Yahoo Finance Implementation
### 1.1 Research & Documentation
- [ ] Research Yahoo Finance API endpoints (no official API, using public endpoints)
- [ ] Document available data fields and formats
- [ ] Identify rate limits and restrictions
- [ ] Test endpoints manually with curl
### 1.2 Module Structure
```
ts/
├── index.ts # Main exports
├── plugins.ts # External dependencies
└── stocks/
├── index.ts # Stocks module exports
├── classes.stockservice.ts # Main StockPriceService class
├── interfaces/
│ ├── stockprice.ts # IStockPrice interface
│ └── provider.ts # IStockProvider interface (for all providers)
└── providers/
├── provider.yahoo.ts # Yahoo Finance implementation
└── (future: provider.alphavantage.ts, provider.iex.ts, etc.)
```
### 1.3 Core Interfaces
```typescript
// IStockPrice - Standardized stock price data
interface IStockPrice {
ticker: string;
price: number;
currency: string;
change: number;
changePercent: number;
timestamp: Date;
provider: string;
}
// IStockProvider - Provider implementation contract
interface IStockProvider {
name: string;
fetchPrice(ticker: string): Promise<IStockPrice>;
fetchPrices(tickers: string[]): Promise<IStockPrice[]>;
isAvailable(): Promise<boolean>;
}
```
### 1.4 Yahoo Finance Provider Implementation
- [ ] Create YahooFinanceProvider class
- [ ] Implement HTTP requests to Yahoo Finance endpoints
- [ ] Parse response data into IStockPrice format
- [ ] Handle errors and edge cases
- [ ] Add request throttling/rate limiting
### 1.5 Main Service Class
- [ ] Create StockPriceService class with provider registry
- [ ] Implement provider interface for pluggable providers
- [ ] Register Yahoo provider (with ability to add more later)
- [ ] Add method for single ticker lookup
- [ ] Add method for batch ticker lookup
- [ ] Implement error handling with graceful degradation
- [ ] Design fallback mechanism (ready for multiple providers)
## Phase 2: Core Features
### 2.1 Service Architecture
- [ ] Create provider registry pattern for managing multiple providers
- [ ] Implement provider priority and selection logic
- [ ] Design provider health check interface
- [ ] Create provider configuration system
- [ ] Implement provider discovery mechanism
- [ ] Add provider capability querying (which tickers/markets supported)
## Phase 3: Advanced Features
### 3.1 Caching System
- [ ] Design cache interface
- [ ] Implement in-memory cache with TTL
- [ ] Add cache invalidation logic
- [ ] Make cache configurable per ticker
### 3.2 Configuration
- [ ] Provider configuration (timeout, retry settings)
- [ ] Cache configuration (TTL, max entries)
- [ ] Request timeout configuration
- [ ] Error handling configuration
### 3.3 Error Handling
- [ ] Define custom error types
- [ ] Implement retry logic with exponential backoff
- [ ] Add circuit breaker pattern for failing providers
- [ ] Comprehensive error logging
## Phase 4: Testing
### 4.1 Unit Tests
- [ ] Test each provider independently
- [ ] Mock HTTP requests for predictable testing
- [ ] Test error scenarios
- [ ] Test data transformation logic
### 4.2 Integration Tests
- [ ] Test with real API calls (rate limit aware)
- [ ] Test provider fallback scenarios
- [ ] Test batch operations
- [ ] Test cache behavior
### 4.3 Performance Tests
- [ ] Measure response times
- [ ] Test concurrent request handling
- [ ] Validate cache effectiveness
## Implementation Order
1. **Week 1: Yahoo Finance Provider**
- Research and test Yahoo endpoints
- Implement basic provider and service
- Create core interfaces
- Basic error handling
2. **Week 2: Service Architecture**
- Create extensible provider system
- Implement provider interface
- Add provider registration
3. **Week 3: Advanced Features**
- Implement caching system
- Add configuration management
- Enhance error handling
4. **Week 4: Testing & Documentation**
- Write comprehensive tests
- Create usage documentation
- Performance optimization
## Dependencies
### Required
- `@push.rocks/smartrequest` - HTTP requests
- `@push.rocks/smartpromise` - Promise utilities
- `@push.rocks/smartlog` - Logging
### Development
- `@git.zone/tstest` - Testing framework
- `@git.zone/tsrun` - TypeScript execution
## API Endpoints Research
### Yahoo Finance
- Base URL: `https://query1.finance.yahoo.com/v8/finance/chart/{ticker}`
- No authentication required
- Returns JSON with price data
- Rate limits unknown (need to test)
- Alternative endpoints to explore:
- `/v7/finance/quote` - Simplified quote data
- `/v10/finance/quoteSummary` - Detailed company data
## Success Criteria
1. Can fetch current stock prices for any valid ticker
2. Extensible architecture for future providers
3. Response time < 1 second for cached data
4. Response time < 3 seconds for fresh data
5. Proper error handling and recovery
6. Comprehensive test coverage (>80%)
## Notes
- Yahoo Finance provides free stock data without authentication
- **Architecture designed for multiple providers**: While only implementing Yahoo Finance initially, all interfaces, classes, and patterns are designed to support multiple stock data providers
- The provider registry pattern allows adding new providers without modifying existing code
- Each provider implements the same IStockProvider interface for consistency
- Future providers can be added by simply creating a new provider class and registering it
- Implement proper TypeScript types for all data structures
- Follow the project's coding standards (prefix interfaces with 'I')
- Use plugins.ts for all external dependencies
- Keep filenames lowercase
- Write tests using @git.zone/tstest with smartexpect syntax
- Focus on clean, extensible architecture for future growth
## Future Provider Addition Example
When ready to add a new provider (e.g., Alpha Vantage), the process will be:
1. Create `ts/stocks/providers/provider.alphavantage.ts`
2. Implement the `IStockProvider` interface
3. Register the provider in the StockPriceService
4. No changes needed to existing code or interfaces

View File

@@ -0,0 +1,395 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
/**
* Test to inspect actual cache contents and verify data integrity
*/
class MockProvider implements opendata.IStockProvider {
name = 'MockProvider';
priority = 100;
requiresAuth = false;
public callLog: Array<{ type: string; ticker: string; timestamp: Date }> = [];
async fetchData(request: opendata.IStockDataRequest): Promise<opendata.IStockPrice | opendata.IStockPrice[]> {
this.callLog.push({
type: request.type,
ticker: request.type === 'batch' ? request.tickers.join(',') : (request as any).ticker,
timestamp: new Date()
});
if (request.type === 'intraday') {
const count = request.limit || 10;
const prices: opendata.IStockPrice[] = [];
const baseTime = request.date || new Date('2025-01-07T09:30:00.000Z');
for (let i = 0; i < count; i++) {
prices.push({
ticker: request.ticker,
price: 100 + i,
currency: 'USD',
timestamp: new Date(baseTime.getTime() + i * 60 * 1000),
fetchedAt: new Date(),
provider: this.name,
dataType: 'intraday',
marketState: 'REGULAR',
open: 100,
high: 101,
low: 99,
volume: 1000000,
change: 0,
changePercent: 0,
previousClose: 100
});
}
return prices;
}
// Default single price
return {
ticker: (request as any).ticker,
price: 150,
currency: 'USD',
timestamp: new Date(),
fetchedAt: new Date(),
provider: this.name,
dataType: 'eod',
marketState: 'CLOSED',
open: 149,
high: 151,
low: 148,
volume: 5000000,
change: 1,
changePercent: 0.67,
previousClose: 149
};
}
async isAvailable(): Promise<boolean> {
return true;
}
}
let stockService: opendata.StockPriceService;
let mockProvider: MockProvider;
tap.test('Cache Inspection - Setup', async () => {
stockService = new opendata.StockPriceService({
ttl: 60000,
maxEntries: 100
});
mockProvider = new MockProvider();
stockService.register(mockProvider);
console.log('✓ Service and provider initialized');
});
tap.test('Cache Inspection - Verify Cache Key Generation', async () => {
await tap.test('should generate unique cache keys for different requests', async () => {
stockService.clearCache();
mockProvider.callLog = [];
// Fetch with different parameters
await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 });
await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 20 });
await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '5min', limit: 10 });
await stockService.getData({ type: 'intraday', ticker: 'MSFT', interval: '1min', limit: 10 });
// Should have made 4 provider calls (all different cache keys)
expect(mockProvider.callLog.length).toEqual(4);
console.log('✓ Cache keys are unique for different parameters');
console.log(` Total provider calls: ${mockProvider.callLog.length}`);
});
await tap.test('should reuse cache for identical requests', async () => {
stockService.clearCache();
mockProvider.callLog = [];
// Same request 3 times
const result1 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 });
const result2 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 });
const result3 = await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 10 });
// Should have made only 1 provider call
expect(mockProvider.callLog.length).toEqual(1);
// All results should be identical (same reference from cache)
expect((result1 as opendata.IStockPrice[]).length).toEqual((result2 as opendata.IStockPrice[]).length);
expect((result1 as opendata.IStockPrice[]).length).toEqual((result3 as opendata.IStockPrice[]).length);
// Verify timestamps match (exact same cached data)
const ts1 = (result1 as opendata.IStockPrice[])[0].timestamp.getTime();
const ts2 = (result2 as opendata.IStockPrice[])[0].timestamp.getTime();
const ts3 = (result3 as opendata.IStockPrice[])[0].timestamp.getTime();
expect(ts1).toEqual(ts2);
expect(ts2).toEqual(ts3);
console.log('✓ Cache reused for identical requests');
console.log(` 3 requests → 1 provider call`);
});
});
tap.test('Cache Inspection - Verify Data Structure', async () => {
await tap.test('should cache complete IStockPrice objects', async () => {
stockService.clearCache();
const result = await stockService.getData({
type: 'intraday',
ticker: 'TSLA',
interval: '1min',
limit: 5
});
expect(result).toBeArray();
const prices = result as opendata.IStockPrice[];
// Verify structure of cached data
for (const price of prices) {
expect(price).toHaveProperty('ticker');
expect(price).toHaveProperty('price');
expect(price).toHaveProperty('currency');
expect(price).toHaveProperty('timestamp');
expect(price).toHaveProperty('fetchedAt');
expect(price).toHaveProperty('provider');
expect(price).toHaveProperty('dataType');
expect(price).toHaveProperty('marketState');
expect(price).toHaveProperty('open');
expect(price).toHaveProperty('high');
expect(price).toHaveProperty('low');
expect(price).toHaveProperty('volume');
// Verify types
expect(typeof price.ticker).toEqual('string');
expect(typeof price.price).toEqual('number');
expect(price.timestamp).toBeInstanceOf(Date);
expect(price.fetchedAt).toBeInstanceOf(Date);
}
console.log('✓ Cached data has complete IStockPrice structure');
console.log(` Sample: ${prices[0].ticker} @ $${prices[0].price} (${prices[0].timestamp.toISOString()})`);
});
await tap.test('should preserve array order in cache', async () => {
stockService.clearCache();
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 10
});
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 10
});
const prices1 = result1 as opendata.IStockPrice[];
const prices2 = result2 as opendata.IStockPrice[];
// Verify order is preserved
for (let i = 0; i < prices1.length; i++) {
expect(prices1[i].timestamp.getTime()).toEqual(prices2[i].timestamp.getTime());
expect(prices1[i].price).toEqual(prices2[i].price);
}
console.log('✓ Array order preserved in cache');
});
});
tap.test('Cache Inspection - Verify TTL Behavior', async () => {
await tap.test('should respect cache TTL for intraday data', async (testArg) => {
// Create service with very short TTL for testing
const shortTTLService = new opendata.StockPriceService({
ttl: 100, // 100ms
maxEntries: 100
});
const testProvider = new MockProvider();
shortTTLService.register(testProvider);
// First fetch
await shortTTLService.getData({
type: 'intraday',
ticker: 'TEST',
interval: '1min',
limit: 5
});
const callCount1 = testProvider.callLog.length;
// Immediate second fetch - should hit cache
await shortTTLService.getData({
type: 'intraday',
ticker: 'TEST',
interval: '1min',
limit: 5
});
const callCount2 = testProvider.callLog.length;
expect(callCount2).toEqual(callCount1); // No new call
// Wait for TTL to expire
await new Promise(resolve => setTimeout(resolve, 150));
// Third fetch - should hit provider (cache expired)
await shortTTLService.getData({
type: 'intraday',
ticker: 'TEST',
interval: '1min',
limit: 5
});
const callCount3 = testProvider.callLog.length;
expect(callCount3).toBeGreaterThan(callCount2); // New call made
console.log('✓ Cache TTL working correctly');
console.log(` Before expiry: ${callCount2 - callCount1} new calls`);
console.log(` After expiry: ${callCount3 - callCount2} new calls`);
});
});
tap.test('Cache Inspection - Memory Efficiency', async () => {
await tap.test('should store deduplicated data in cache', async () => {
stockService.clearCache();
mockProvider.callLog = [];
// Fetch data
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 100
});
const prices = result1 as opendata.IStockPrice[];
// Verify no duplicate timestamps in cached data
const timestamps = prices.map(p => p.timestamp.getTime());
const uniqueTimestamps = new Set(timestamps);
expect(uniqueTimestamps.size).toEqual(timestamps.length);
console.log('✓ No duplicate timestamps in cached data');
console.log(` Records: ${prices.length}`);
console.log(` Unique timestamps: ${uniqueTimestamps.size}`);
});
await tap.test('should estimate memory usage', async () => {
stockService.clearCache();
// Fetch various sizes
await stockService.getData({ type: 'intraday', ticker: 'AAPL', interval: '1min', limit: 100 });
await stockService.getData({ type: 'intraday', ticker: 'MSFT', interval: '1min', limit: 100 });
await stockService.getData({ type: 'intraday', ticker: 'GOOGL', interval: '5min', limit: 50 });
// Estimate memory (rough calculation)
// Each IStockPrice is approximately 300-400 bytes
const totalRecords = 100 + 100 + 50;
const estimatedBytes = totalRecords * 350; // Average 350 bytes per record
const estimatedKB = (estimatedBytes / 1024).toFixed(2);
console.log('✓ Cache memory estimation:');
console.log(` Total records cached: ${totalRecords}`);
console.log(` Estimated memory: ~${estimatedKB} KB`);
console.log(` Average per record: ~350 bytes`);
});
});
tap.test('Cache Inspection - Edge Cases', async () => {
await tap.test('should handle empty results', async () => {
const emptyProvider = new MockProvider();
emptyProvider.fetchData = async () => [];
const emptyService = new opendata.StockPriceService();
emptyService.register(emptyProvider);
const result = await emptyService.getData({
type: 'intraday',
ticker: 'EMPTY',
interval: '1min'
});
expect(result).toBeArray();
expect((result as opendata.IStockPrice[]).length).toEqual(0);
// Second fetch should still hit cache (even though empty)
const result2 = await emptyService.getData({
type: 'intraday',
ticker: 'EMPTY',
interval: '1min'
});
expect(result2).toBeArray();
expect((result2 as opendata.IStockPrice[]).length).toEqual(0);
console.log('✓ Empty results cached correctly');
});
await tap.test('should handle single record', async () => {
stockService.clearCache();
const result = await stockService.getData({
type: 'intraday',
ticker: 'SINGLE',
interval: '1min',
limit: 1
});
expect(result).toBeArray();
expect((result as opendata.IStockPrice[]).length).toEqual(1);
console.log('✓ Single record cached correctly');
});
});
tap.test('Cache Inspection - Verify fetchedAt Timestamps', async () => {
await tap.test('should preserve fetchedAt in cached data', async () => {
stockService.clearCache();
const beforeFetch = Date.now();
const result = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 5
});
const afterFetch = Date.now();
const prices = result as opendata.IStockPrice[];
for (const price of prices) {
const fetchedTime = price.fetchedAt.getTime();
expect(fetchedTime).toBeGreaterThanOrEqual(beforeFetch);
expect(fetchedTime).toBeLessThanOrEqual(afterFetch);
}
// Fetch again - fetchedAt should be the same (from cache)
await new Promise(resolve => setTimeout(resolve, 50)); // Small delay
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 5
});
const prices2 = result2 as opendata.IStockPrice[];
// Verify fetchedAt matches (same cached data)
for (let i = 0; i < prices.length; i++) {
expect(prices2[i].fetchedAt.getTime()).toEqual(prices[i].fetchedAt.getTime());
}
console.log('✓ fetchedAt timestamps preserved in cache');
});
});
export default tap.start();

View File

@@ -0,0 +1,261 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
// Test data
const testCryptos = ['BTC', 'ETH', 'USDT'];
const testCryptoIds = ['bitcoin', 'ethereum', 'tether'];
const invalidCrypto = 'INVALID_CRYPTO_XYZ_12345';
let stockService: opendata.StockPriceService;
let coingeckoProvider: opendata.CoinGeckoProvider;
tap.test('should create StockPriceService instance', async () => {
stockService = new opendata.StockPriceService({
ttl: 30000, // 30 seconds cache
maxEntries: 100
});
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
});
tap.test('should create CoinGeckoProvider instance without API key', async () => {
coingeckoProvider = new opendata.CoinGeckoProvider();
expect(coingeckoProvider).toBeInstanceOf(opendata.CoinGeckoProvider);
expect(coingeckoProvider.name).toEqual('CoinGecko');
expect(coingeckoProvider.requiresAuth).toEqual(false);
expect(coingeckoProvider.priority).toEqual(90);
});
tap.test('should register CoinGecko provider with the service', async () => {
stockService.register(coingeckoProvider);
const providers = stockService.getAllProviders();
expect(providers).toContainEqual(coingeckoProvider);
expect(stockService.getProvider('CoinGecko')).toEqual(coingeckoProvider);
});
tap.test('should check CoinGecko provider health', async () => {
const health = await stockService.checkProvidersHealth();
expect(health.get('CoinGecko')).toEqual(true);
});
tap.test('should fetch single crypto price using ticker symbol (BTC)', async () => {
const price = await stockService.getPrice({ ticker: 'BTC' });
expect(price).toHaveProperty('ticker');
expect(price).toHaveProperty('price');
expect(price).toHaveProperty('currency');
expect(price).toHaveProperty('change');
expect(price).toHaveProperty('changePercent');
expect(price).toHaveProperty('previousClose');
expect(price).toHaveProperty('timestamp');
expect(price).toHaveProperty('provider');
expect(price).toHaveProperty('marketState');
expect(price.ticker).toEqual('BTC');
expect(price.price).toBeGreaterThan(0);
expect(price.currency).toEqual('USD');
expect(price.provider).toEqual('CoinGecko');
expect(price.marketState).toEqual('REGULAR'); // Crypto is 24/7
expect(price.timestamp).toBeInstanceOf(Date);
expect(price.dataType).toEqual('live');
console.log(`\n📊 BTC Price: $${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`);
console.log(` Change: ${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%`);
});
tap.test('should fetch single crypto price using CoinGecko ID (bitcoin)', async () => {
// Clear cache to ensure fresh fetch
stockService.clearCache();
const price = await stockService.getPrice({ ticker: 'bitcoin' });
expect(price.ticker).toEqual('BITCOIN');
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('CoinGecko');
expect(price.companyName).toInclude('Bitcoin');
});
tap.test('should fetch multiple crypto prices (batch)', async () => {
stockService.clearCache();
const prices = await stockService.getPrices({
tickers: testCryptos
});
expect(prices).toBeArray();
expect(prices.length).toEqual(testCryptos.length);
for (const price of prices) {
expect(testCryptos).toContain(price.ticker);
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('CoinGecko');
expect(price.marketState).toEqual('REGULAR');
console.log(`\n💰 ${price.ticker}: $${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 8 })}`);
console.log(` Change 24h: ${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%`);
if (price.volume) {
console.log(` Volume 24h: $${price.volume.toLocaleString('en-US')}`);
}
}
});
tap.test('should fetch historical crypto prices', async () => {
// Add delay to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 3000));
const to = new Date();
const from = new Date(to.getTime() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
const prices = await stockService.getData({
type: 'historical',
ticker: 'BTC',
from: from,
to: to
});
expect(prices).toBeArray();
expect((prices as opendata.IStockPrice[]).length).toBeGreaterThan(0);
const pricesArray = prices as opendata.IStockPrice[];
console.log(`\n📈 Historical BTC Prices (${pricesArray.length} days):`);
// Show first few and last few
const toShow = Math.min(3, pricesArray.length);
for (let i = 0; i < toShow; i++) {
const price = pricesArray[i];
const date = price.timestamp.toISOString().split('T')[0];
console.log(` ${date}: $${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`);
}
if (pricesArray.length > toShow * 2) {
console.log(' ...');
}
for (let i = Math.max(toShow, pricesArray.length - toShow); i < pricesArray.length; i++) {
const price = pricesArray[i];
const date = price.timestamp.toISOString().split('T')[0];
console.log(` ${date}: $${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`);
}
// Validate first entry
const firstPrice = pricesArray[0];
expect(firstPrice.ticker).toEqual('BTC');
expect(firstPrice.dataType).toEqual('eod');
expect(firstPrice.provider).toEqual('CoinGecko');
});
tap.test('should fetch intraday crypto prices (hourly)', async () => {
// Add delay to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 3000));
const prices = await stockService.getData({
type: 'intraday',
ticker: 'ETH',
interval: '1hour',
limit: 12 // Last 12 hours
});
expect(prices).toBeArray();
expect((prices as opendata.IStockPrice[]).length).toBeGreaterThan(0);
const pricesArray = prices as opendata.IStockPrice[];
console.log(`\n⏰ Intraday ETH Prices (hourly, last ${pricesArray.length} hours):`);
// Show first few entries
const toShow = Math.min(5, pricesArray.length);
for (let i = 0; i < toShow; i++) {
const price = pricesArray[i];
const time = price.timestamp.toISOString().replace('T', ' ').substring(0, 16);
console.log(` ${time}: $${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`);
}
// Validate first entry
const firstPrice = pricesArray[0];
expect(firstPrice.ticker).toEqual('ETH');
expect(firstPrice.dataType).toEqual('intraday');
expect(firstPrice.provider).toEqual('CoinGecko');
});
tap.test('should serve cached prices on subsequent requests', async () => {
// First request - should hit the API
const firstRequest = await stockService.getPrice({ ticker: 'BTC' });
// Second request - should be served from cache
const secondRequest = await stockService.getPrice({ ticker: 'BTC' });
expect(secondRequest.ticker).toEqual(firstRequest.ticker);
expect(secondRequest.price).toEqual(firstRequest.price);
expect(secondRequest.timestamp).toEqual(firstRequest.timestamp);
expect(secondRequest.fetchedAt).toEqual(firstRequest.fetchedAt);
});
tap.test('should handle invalid crypto ticker gracefully', async () => {
try {
await stockService.getPrice({ ticker: invalidCrypto });
throw new Error('Should have thrown an error for invalid ticker');
} catch (error) {
expect(error.message).toInclude('Failed to fetch');
}
});
tap.test('should support market checking', async () => {
expect(coingeckoProvider.supportsMarket('CRYPTO')).toEqual(true);
expect(coingeckoProvider.supportsMarket('BTC')).toEqual(true);
expect(coingeckoProvider.supportsMarket('ETH')).toEqual(true);
expect(coingeckoProvider.supportsMarket('NASDAQ')).toEqual(false);
});
tap.test('should support ticker validation', async () => {
expect(coingeckoProvider.supportsTicker('BTC')).toEqual(true);
expect(coingeckoProvider.supportsTicker('bitcoin')).toEqual(true);
expect(coingeckoProvider.supportsTicker('wrapped-bitcoin')).toEqual(true);
expect(coingeckoProvider.supportsTicker('BTC!')).toEqual(false);
expect(coingeckoProvider.supportsTicker('BTC@USD')).toEqual(false);
});
tap.test('should display provider statistics', async () => {
const stats = stockService.getProviderStats();
const coingeckoStats = stats.get('CoinGecko');
expect(coingeckoStats).toBeTruthy();
expect(coingeckoStats.successCount).toBeGreaterThan(0);
console.log('\n📊 CoinGecko Provider Statistics:');
console.log(` Success Count: ${coingeckoStats.successCount}`);
console.log(` Error Count: ${coingeckoStats.errorCount}`);
if (coingeckoStats.lastError) {
console.log(` Last Error: ${coingeckoStats.lastError}`);
}
});
tap.test('should display crypto price dashboard', async () => {
// Add delay to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 3000));
stockService.clearCache();
const cryptos = ['BTC', 'ETH', 'BNB', 'SOL', 'ADA'];
const prices = await stockService.getPrices({ tickers: cryptos });
console.log('\n╔═══════════════════════════════════════════════════════════╗');
console.log('║ 🌐 CRYPTOCURRENCY PRICE DASHBOARD ║');
console.log('╠═══════════════════════════════════════════════════════════╣');
for (const price of prices) {
const priceStr = `$${price.price.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 8 })}`;
const changeStr = `${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%`;
const changeIcon = price.changePercent >= 0 ? '📈' : '📉';
console.log(`${price.ticker.padEnd(6)} ${changeIcon} ${priceStr.padStart(20)}${changeStr.padStart(10)}`);
}
console.log('╚═══════════════════════════════════════════════════════════╝');
console.log(`Provider: ${prices[0].provider} | Market State: ${prices[0].marketState} (24/7)`);
console.log(`Fetched at: ${prices[0].fetchedAt.toISOString()}`);
});
tap.test('should clear cache', async () => {
stockService.clearCache();
// Cache is cleared, no assertions needed
});
export default tap.start();

View File

@@ -0,0 +1,287 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
const TEST_USER_AGENT = 'fin.cx test@fin.cx';
tap.test('FundamentalsService - Provider Registration', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
await tap.test('should register provider', async () => {
service.register(provider);
const registered = service.getProvider('SEC EDGAR');
expect(registered).toBeDefined();
expect(registered?.name).toEqual('SEC EDGAR');
});
await tap.test('should get all providers', async () => {
const providers = service.getAllProviders();
expect(providers.length).toBeGreaterThan(0);
expect(providers[0].name).toEqual('SEC EDGAR');
});
await tap.test('should get enabled providers', async () => {
const providers = service.getEnabledProviders();
expect(providers.length).toBeGreaterThan(0);
});
await tap.test('should unregister provider', async () => {
service.unregister('SEC EDGAR');
const registered = service.getProvider('SEC EDGAR');
expect(registered).toBeUndefined();
// Re-register for other tests
service.register(provider);
});
});
tap.test('FundamentalsService - Fetch Fundamentals', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should fetch fundamentals for single ticker', async () => {
const fundamentals = await service.getFundamentals('AAPL');
expect(fundamentals).toBeDefined();
expect(fundamentals.ticker).toEqual('AAPL');
expect(fundamentals.companyName).toEqual('Apple Inc.');
expect(fundamentals.provider).toEqual('SEC EDGAR');
expect(fundamentals.earningsPerShareDiluted).toBeGreaterThan(0);
expect(fundamentals.sharesOutstanding).toBeGreaterThan(0);
console.log('\n📊 Fetched via Service:');
console.log(` ${fundamentals.ticker}: ${fundamentals.companyName}`);
console.log(` EPS: $${fundamentals.earningsPerShareDiluted?.toFixed(2)}`);
console.log(` Shares: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
});
await tap.test('should fetch fundamentals for multiple tickers', async () => {
const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']);
expect(fundamentalsList).toBeInstanceOf(Array);
expect(fundamentalsList.length).toEqual(2);
const apple = fundamentalsList.find(f => f.ticker === 'AAPL');
const msft = fundamentalsList.find(f => f.ticker === 'MSFT');
expect(apple).toBeDefined();
expect(msft).toBeDefined();
expect(apple!.companyName).toEqual('Apple Inc.');
expect(msft!.companyName).toContain('Microsoft');
console.log('\n📊 Batch Fetch via Service:');
fundamentalsList.forEach(f => {
console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`);
});
});
});
tap.test('FundamentalsService - Caching', async () => {
const service = new opendata.FundamentalsService({
ttl: 60000, // 60 seconds for testing
maxEntries: 100
});
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should cache fundamentals data', async () => {
// Clear cache first
service.clearCache();
let stats = service.getCacheStats();
expect(stats.size).toEqual(0);
// First fetch (should hit API)
const start1 = Date.now();
await service.getFundamentals('AAPL');
const duration1 = Date.now() - start1;
stats = service.getCacheStats();
expect(stats.size).toEqual(1);
// Second fetch (should hit cache - much faster)
const start2 = Date.now();
await service.getFundamentals('AAPL');
const duration2 = Date.now() - start2;
expect(duration2).toBeLessThan(duration1);
console.log('\n⚡ Cache Performance:');
console.log(` First fetch: ${duration1}ms`);
console.log(` Cached fetch: ${duration2}ms`);
console.log(` Speedup: ${Math.round(duration1 / duration2)}x`);
});
await tap.test('should respect cache TTL', async () => {
// Set very short TTL
service.setCacheTTL(100); // 100ms
// Fetch and cache
await service.getFundamentals('MSFT');
// Wait for TTL to expire
await new Promise(resolve => setTimeout(resolve, 150));
// This should fetch again (cache expired)
const stats = service.getCacheStats();
console.log(`\n Cache TTL: ${stats.ttl}ms`);
});
await tap.test('should clear cache', async () => {
service.clearCache();
const stats = service.getCacheStats();
expect(stats.size).toEqual(0);
});
});
tap.test('FundamentalsService - Price Enrichment', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should enrich fundamentals with price to calculate market cap', async () => {
const fundamentals = await service.getFundamentals('AAPL');
// Simulate current price
const currentPrice = 270.37;
const enriched = await service.enrichWithPrice(fundamentals, currentPrice);
expect(enriched.marketCap).toBeDefined();
expect(enriched.priceToEarnings).toBeDefined();
expect(enriched.priceToBook).toBeDefined();
expect(enriched.marketCap).toBeGreaterThan(0);
expect(enriched.priceToEarnings).toBeGreaterThan(0);
console.log('\n💰 Enriched with Price ($270.37):');
console.log(` Market Cap: $${(enriched.marketCap! / 1_000_000_000_000).toFixed(2)}T`);
console.log(` P/E Ratio: ${enriched.priceToEarnings!.toFixed(2)}`);
console.log(` P/B Ratio: ${enriched.priceToBook?.toFixed(2) || 'N/A'}`);
// Verify calculations
const expectedMarketCap = fundamentals.sharesOutstanding! * currentPrice;
expect(Math.abs(enriched.marketCap! - expectedMarketCap)).toBeLessThan(1); // Allow for rounding
const expectedPE = currentPrice / fundamentals.earningsPerShareDiluted!;
expect(Math.abs(enriched.priceToEarnings! - expectedPE)).toBeLessThan(0.01);
});
await tap.test('should enrich batch fundamentals with prices', async () => {
const fundamentalsList = await service.getBatchFundamentals(['AAPL', 'MSFT']);
const priceMap = new Map<string, number>([
['AAPL', 270.37],
['MSFT', 425.50]
]);
const enriched = await service.enrichBatchWithPrices(fundamentalsList, priceMap);
expect(enriched.length).toEqual(2);
const apple = enriched.find(f => f.ticker === 'AAPL')!;
const msft = enriched.find(f => f.ticker === 'MSFT')!;
expect(apple.marketCap).toBeGreaterThan(0);
expect(msft.marketCap).toBeGreaterThan(0);
console.log('\n💰 Batch Enrichment:');
console.log(` AAPL: Market Cap $${(apple.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${apple.priceToEarnings!.toFixed(2)}`);
console.log(` MSFT: Market Cap $${(msft.marketCap! / 1_000_000_000_000).toFixed(2)}T, P/E ${msft.priceToEarnings!.toFixed(2)}`);
});
});
tap.test('FundamentalsService - Provider Health', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should check provider health', async () => {
const health = await service.checkProvidersHealth();
expect(health.size).toEqual(1);
expect(health.get('SEC EDGAR')).toBe(true);
console.log('\n💚 Provider Health:');
health.forEach((isHealthy, name) => {
console.log(` ${name}: ${isHealthy ? '✅ Healthy' : '❌ Unhealthy'}`);
});
});
});
tap.test('FundamentalsService - Provider Statistics', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should track provider statistics', async () => {
// Make some requests
await service.getFundamentals('AAPL');
await service.getFundamentals('MSFT');
const stats = service.getProviderStats();
expect(stats.size).toEqual(1);
const secStats = stats.get('SEC EDGAR');
expect(secStats).toBeDefined();
expect(secStats!.successCount).toBeGreaterThan(0);
console.log('\n📈 Provider Stats:');
console.log(` Success Count: ${secStats!.successCount}`);
console.log(` Error Count: ${secStats!.errorCount}`);
});
});
tap.test('FundamentalsService - Error Handling', async () => {
const service = new opendata.FundamentalsService();
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.register(provider);
await tap.test('should throw error for invalid ticker', async () => {
try {
await service.getFundamentals('INVALIDTICKER123456');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('CIK not found');
}
});
await tap.test('should throw error when no providers available', async () => {
const emptyService = new opendata.FundamentalsService();
try {
await emptyService.getFundamentals('AAPL');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('No fundamentals providers available');
}
});
});
export default tap.start();

View File

@@ -0,0 +1,54 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js'
import * as paths from '../ts/paths.js';
import * as plugins from '../ts/plugins.js';
import { BusinessRecord } from '../ts/classes.businessrecord.js';
// Test configuration - explicit paths required
const testNogitDir = plugins.path.join(paths.packageDir, '.nogit');
const testDownloadDir = plugins.path.join(testNogitDir, 'downloads');
const testGermanBusinessDataDir = plugins.path.join(testNogitDir, 'germanbusinessdata');
const testOutputDir = plugins.path.join(testNogitDir, 'testoutput');
let testOpenDataInstance: opendata.OpenData;
tap.test('first test', async () => {
testOpenDataInstance = new opendata.OpenData({
nogitDir: testNogitDir,
downloadDir: testDownloadDir,
germanBusinessDataDir: testGermanBusinessDataDir
});
expect(testOpenDataInstance).toBeInstanceOf(opendata.OpenData);
});
tap.test('should start the instance', async () => {
await testOpenDataInstance.start();
});
const resultsSearch = tap.test('should get the data for a company', async () => {
const result = await testOpenDataInstance.handelsregister.searchCompany('LADR', 20);
console.log(result);
return result;
});
tap.test('should get the data for a specific company', async () => {
let testCompany: BusinessRecord['data']['germanParsedRegistration'] = (await resultsSearch.testResultPromise)[0]['germanParsedRegistration'];
console.log(`trying to find specific company with:`);
console.log(testCompany);
const result = await testOpenDataInstance.handelsregister.getSpecificCompany(testCompany);
console.log(result);
await Promise.all(result.files.map(async (file) => {
await file.writeToDir(testOutputDir);
}));
});
tap.test('should stop the instance', async (toolsArg) => {
await testOpenDataInstance.stop();
});
tap.start()

View File

@@ -0,0 +1,582 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
/**
* Mock provider for testing incremental cache behavior
* Allows precise control over what data is returned to test cache logic
*/
class MockIntradayProvider implements opendata.IStockProvider {
name = 'MockIntraday';
priority = 100;
requiresAuth = false;
// Track fetch calls for testing
public fetchCallCount = 0;
public lastRequest: opendata.IStockDataRequest | null = null;
// Mock data to return
private mockData: opendata.IStockPrice[] = [];
/**
* Set the mock data that will be returned on next fetch
*/
public setMockData(data: opendata.IStockPrice[]): void {
this.mockData = data;
}
/**
* Reset fetch tracking
*/
public resetTracking(): void {
this.fetchCallCount = 0;
this.lastRequest = null;
}
async fetchData(request: opendata.IStockDataRequest): Promise<opendata.IStockPrice | opendata.IStockPrice[]> {
this.fetchCallCount++;
this.lastRequest = request;
// For intraday requests, return filtered data based on date
if (request.type === 'intraday') {
let filteredData = [...this.mockData];
// Filter by date if specified (simulate incremental fetch)
if (request.date) {
filteredData = filteredData.filter(p => p.timestamp > request.date!);
}
// Apply limit
if (request.limit) {
filteredData = filteredData.slice(-request.limit);
}
return filteredData;
}
// For other requests, return first item or empty array
if (this.mockData.length > 0) {
return this.mockData[0];
}
throw new Error('No mock data available');
}
async isAvailable(): Promise<boolean> {
return true;
}
}
/**
* Helper to generate mock intraday prices
*/
function generateMockIntradayPrices(
ticker: string,
count: number,
startTime: Date,
intervalMinutes: number = 1
): opendata.IStockPrice[] {
const prices: opendata.IStockPrice[] = [];
let basePrice = 100;
for (let i = 0; i < count; i++) {
const timestamp = new Date(startTime.getTime() + i * intervalMinutes * 60 * 1000);
basePrice += (Math.random() - 0.5) * 2; // Random walk
prices.push({
ticker,
price: basePrice,
currency: 'USD',
timestamp,
fetchedAt: new Date(),
provider: 'MockIntraday',
dataType: 'intraday',
marketState: 'REGULAR',
open: basePrice - 0.5,
high: basePrice + 1,
low: basePrice - 1,
volume: 1000000,
change: 0,
changePercent: 0,
previousClose: basePrice
});
}
return prices;
}
let stockService: opendata.StockPriceService;
let mockProvider: MockIntradayProvider;
tap.test('Incremental Cache Setup', async () => {
await tap.test('should create StockPriceService and MockProvider', async () => {
stockService = new opendata.StockPriceService({
ttl: 60000, // 1 minute default (will be overridden by smart TTL)
maxEntries: 1000
});
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
mockProvider = new MockIntradayProvider();
stockService.register(mockProvider);
const providers = stockService.getEnabledProviders();
expect(providers).toContainEqual(mockProvider);
console.log('✓ Test setup complete');
});
});
tap.test('Incremental Cache - Basic Behavior', async () => {
await tap.test('should cache intraday data on first fetch', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
const mockData = generateMockIntradayPrices('AAPL', 10, startTime, 1);
mockProvider.setMockData(mockData);
// First fetch - should hit provider
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 10
});
expect(result1).toBeArray();
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCallCount).toEqual(1);
console.log('✓ First fetch cached 10 records');
});
await tap.test('should serve from cache on second identical request', async () => {
mockProvider.resetTracking();
// Second fetch - should hit cache (no provider call)
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 10
});
expect(result2).toBeArray();
expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCallCount).toEqual(0); // Should NOT call provider
console.log('✓ Second fetch served from cache (0 provider calls)');
});
});
tap.test('Incremental Cache - Incremental Fetch', async () => {
await tap.test('should only fetch NEW data on refresh', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// First fetch: 10 records from 9:30-9:39
const mockData1 = generateMockIntradayPrices('MSFT', 10, startTime, 1);
mockProvider.setMockData(mockData1);
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'MSFT',
interval: '1min'
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCallCount).toEqual(1);
const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
console.log(`✓ First fetch: 10 records, latest timestamp: ${latestTimestamp1.toISOString()}`);
// Simulate 5 minutes passing - 5 new records available
mockProvider.resetTracking();
const mockData2 = generateMockIntradayPrices('MSFT', 15, startTime, 1); // 15 total (10 old + 5 new)
mockProvider.setMockData(mockData2);
// Second fetch - should detect cache and only fetch NEW data
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'MSFT',
interval: '1min'
});
expect((result2 as opendata.IStockPrice[]).length).toEqual(15);
expect(mockProvider.fetchCallCount).toEqual(1); // Should call provider
// Verify the request had a date filter (incremental fetch)
expect(mockProvider.lastRequest).not.toEqual(null);
expect(mockProvider.lastRequest!.type).toEqual('intraday');
expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined);
const requestDate = (mockProvider.lastRequest as opendata.IStockIntradayRequest).date;
console.log(`✓ Incremental fetch requested data since: ${requestDate!.toISOString()}`);
console.log(`✓ Total records after merge: ${(result2 as opendata.IStockPrice[]).length}`);
console.log('✓ Only fetched NEW data (incremental fetch working)');
});
await tap.test('should return cached data when no new records available', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
const mockData = generateMockIntradayPrices('GOOGL', 10, startTime, 1);
mockProvider.setMockData(mockData);
// First fetch
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min'
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
// Second fetch - same data (no new records)
mockProvider.resetTracking();
mockProvider.setMockData(mockData); // Same data
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min'
});
expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch attempted
console.log('✓ No new records - returned cached data');
});
});
tap.test('Incremental Cache - Deduplication', async () => {
await tap.test('should deduplicate by timestamp in merged data', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// First fetch: 10 records
const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1);
mockProvider.setMockData(mockData1);
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'TSLA',
interval: '1min'
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
// Second fetch: Return overlapping data (last 5 old + 5 new)
// This simulates provider returning some duplicate timestamps
mockProvider.resetTracking();
const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1);
mockProvider.setMockData(mockData2);
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'TSLA',
interval: '1min'
});
// Should have 15 unique timestamps (deduplication worked)
expect((result2 as opendata.IStockPrice[]).length).toEqual(15);
// Verify timestamps are unique
const timestamps = (result2 as opendata.IStockPrice[]).map(p => p.timestamp.getTime());
const uniqueTimestamps = new Set(timestamps);
expect(uniqueTimestamps.size).toEqual(15);
console.log('✓ Deduplication working - 15 unique timestamps');
});
});
tap.test('Incremental Cache - Limit Handling', async () => {
await tap.test('should respect limit parameter in merged results', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// First fetch with limit 100
const mockData1 = generateMockIntradayPrices('AMZN', 100, startTime, 1);
mockProvider.setMockData(mockData1);
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AMZN',
interval: '1min',
limit: 100
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(100);
// Second fetch: 10 new records available
mockProvider.resetTracking();
const mockData2 = generateMockIntradayPrices('AMZN', 110, startTime, 1);
mockProvider.setMockData(mockData2);
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'AMZN',
interval: '1min',
limit: 100 // Same limit
});
// Should still return 100 (most recent 100 after merge)
expect((result2 as opendata.IStockPrice[]).length).toEqual(100);
// Verify we got the most RECENT 100 (should include new data)
const lastTimestamp = (result2 as opendata.IStockPrice[])[99].timestamp;
const expectedLastTimestamp = mockData2[109].timestamp;
expect(lastTimestamp.getTime()).toEqual(expectedLastTimestamp.getTime());
console.log('✓ Limit respected - returned most recent 100 records');
});
await tap.test('should handle different limits without cache collision', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
const mockData = generateMockIntradayPrices('NVDA', 1000, startTime, 1);
mockProvider.setMockData(mockData);
// Fetch with limit 100
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'NVDA',
interval: '1min',
limit: 100
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(100);
mockProvider.resetTracking();
// Fetch with limit 500 (should NOT use cached limit:100 data)
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'NVDA',
interval: '1min',
limit: 500
});
expect((result2 as opendata.IStockPrice[]).length).toEqual(500);
// Should have made a new provider call (different cache key)
expect(mockProvider.fetchCallCount).toBeGreaterThan(0);
console.log('✓ Different limits use different cache keys');
});
});
tap.test('Incremental Cache - Dashboard Polling Scenario', async () => {
await tap.test('should efficiently handle repeated polling requests', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
let currentDataSize = 100;
// Initial fetch: 100 records
let mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1);
mockProvider.setMockData(mockData);
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 1000
});
expect((result1 as opendata.IStockPrice[]).length).toEqual(100);
const initialFetchCount = mockProvider.fetchCallCount;
console.log(`✓ Initial fetch: ${(result1 as opendata.IStockPrice[]).length} records (${initialFetchCount} API calls)`);
// Simulate 5 dashboard refreshes (1 new record each time)
let totalNewRecords = 0;
for (let i = 0; i < 5; i++) {
mockProvider.resetTracking();
currentDataSize += 1; // 1 new record
totalNewRecords += 1;
mockData = generateMockIntradayPrices('AAPL', currentDataSize, startTime, 1);
mockProvider.setMockData(mockData);
const result = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 1000
});
expect((result as opendata.IStockPrice[]).length).toEqual(currentDataSize);
expect(mockProvider.fetchCallCount).toEqual(1); // Incremental fetch
}
console.log(`✓ Dashboard polling: 5 refreshes with ${totalNewRecords} new records`);
console.log('✓ Each refresh only fetched NEW data (incremental cache working)');
});
});
tap.test('Incremental Cache - Memory Impact', async () => {
await tap.test('should demonstrate memory savings from deduplication', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// Create data with intentional duplicates
const baseData = generateMockIntradayPrices('MSFT', 1000, startTime, 1);
const duplicatedData = [...baseData, ...baseData.slice(-100)]; // Duplicate last 100
expect(duplicatedData.length).toEqual(1100); // Before deduplication
mockProvider.setMockData(duplicatedData);
const result = await stockService.getData({
type: 'intraday',
ticker: 'MSFT',
interval: '1min'
});
// Should have 1000 unique records (100 duplicates removed)
expect((result as opendata.IStockPrice[]).length).toEqual(1000);
console.log('✓ Deduplication removed 100 duplicate timestamps');
console.log(`✓ Memory saved: ~${Math.round((100 / 1100) * 100)}%`);
});
});
tap.test('Incremental Cache - Fallback Behavior', async () => {
await tap.test('should not use incremental fetch for requests with date filter', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
const mockData = generateMockIntradayPrices('GOOGL', 100, startTime, 1);
mockProvider.setMockData(mockData);
// First fetch without date
await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min'
});
mockProvider.resetTracking();
// Second fetch WITH date filter - should NOT use incremental cache
const result = await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min',
date: new Date('2025-01-07T10:00:00.000Z') // Explicit date filter
});
// Should have made normal fetch (not incremental)
expect(mockProvider.fetchCallCount).toEqual(1);
expect((mockProvider.lastRequest as opendata.IStockIntradayRequest).date).not.toEqual(undefined);
console.log('✓ Incremental cache skipped for requests with explicit date filter');
});
});
tap.test('Incremental Cache - Performance Benchmark', async () => {
await tap.test('should demonstrate API call reduction', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// Initial dataset: 1000 records
let mockData = generateMockIntradayPrices('BENCHMARK', 1000, startTime, 1);
mockProvider.setMockData(mockData);
// Initial fetch
await stockService.getData({
type: 'intraday',
ticker: 'BENCHMARK',
interval: '1min',
limit: 1000
});
expect(mockProvider.fetchCallCount).toEqual(1);
console.log('✓ Initial fetch: 1000 records');
let totalProviderCalls = 1;
let totalNewRecords = 0;
// Simulate 10 refreshes (5 new records each)
for (let i = 0; i < 10; i++) {
mockProvider.resetTracking();
// Add 5 new records
const newCount = 5;
mockData = generateMockIntradayPrices('BENCHMARK', 1000 + totalNewRecords + newCount, startTime, 1);
mockProvider.setMockData(mockData);
await stockService.getData({
type: 'intraday',
ticker: 'BENCHMARK',
interval: '1min',
limit: 1000
});
totalProviderCalls += mockProvider.fetchCallCount;
totalNewRecords += newCount;
}
console.log('\n📊 Performance Benchmark:');
console.log(` Total refreshes: 10`);
console.log(` New records fetched: ${totalNewRecords}`);
console.log(` Total provider calls: ${totalProviderCalls}`);
console.log(` Without incremental cache: ${11} calls (1 initial + 10 full refreshes)`);
console.log(` With incremental cache: ${totalProviderCalls} calls (1 initial + 10 incremental)`);
console.log(` Data transfer reduction: ~${Math.round((1 - (totalNewRecords / (10 * 1000))) * 100)}%`);
console.log(' (Only fetched NEW data instead of refetching all 1000 records each time)');
});
});
tap.test('Incremental Cache - Timestamp Ordering', async () => {
await tap.test('should maintain timestamp order after merge', async () => {
stockService.clearCache();
mockProvider.resetTracking();
const startTime = new Date('2025-01-07T09:30:00.000Z');
// First fetch
const mockData1 = generateMockIntradayPrices('TSLA', 10, startTime, 1);
mockProvider.setMockData(mockData1);
await stockService.getData({
type: 'intraday',
ticker: 'TSLA',
interval: '1min'
});
// Second fetch with new data
mockProvider.resetTracking();
const mockData2 = generateMockIntradayPrices('TSLA', 15, startTime, 1);
mockProvider.setMockData(mockData2);
const result = await stockService.getData({
type: 'intraday',
ticker: 'TSLA',
interval: '1min'
});
// Verify ascending timestamp order
const timestamps = (result as opendata.IStockPrice[]).map(p => p.timestamp.getTime());
for (let i = 1; i < timestamps.length; i++) {
expect(timestamps[i]).toBeGreaterThan(timestamps[i - 1]);
}
console.log('✓ Timestamps correctly ordered (ascending)');
});
});
export default tap.start();

View File

@@ -0,0 +1,572 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
import * as paths from '../ts/paths.js';
import * as plugins from '../ts/plugins.js';
// Test configuration - explicit paths required
const testNogitDir = plugins.path.join(paths.packageDir, '.nogit');
// Test data
const testTickers = ['AAPL', 'MSFT', 'GOOGL'];
const invalidTicker = 'INVALID_TICKER_XYZ';
let stockService: opendata.StockPriceService;
let marketstackProvider: opendata.MarketstackProvider;
let testQenv: plugins.qenv.Qenv;
tap.test('should create StockPriceService instance', async () => {
stockService = new opendata.StockPriceService({
ttl: 30000, // 30 seconds cache
maxEntries: 100
});
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
});
tap.test('should create MarketstackProvider instance', async () => {
try {
// Create qenv and get API key
testQenv = new plugins.qenv.Qenv(paths.packageDir, testNogitDir);
const apiKey = await testQenv.getEnvVarOnDemand('MARKETSTACK_COM_TOKEN');
marketstackProvider = new opendata.MarketstackProvider(apiKey, {
enabled: true,
timeout: 10000,
retryAttempts: 2,
retryDelay: 500
});
expect(marketstackProvider).toBeInstanceOf(opendata.MarketstackProvider);
expect(marketstackProvider.name).toEqual('Marketstack');
expect(marketstackProvider.requiresAuth).toEqual(true);
expect(marketstackProvider.priority).toEqual(80);
} catch (error) {
if (error.message.includes('MARKETSTACK_COM_TOKEN')) {
console.log('⚠️ MARKETSTACK_COM_TOKEN not set - skipping Marketstack tests');
tap.test('Marketstack token not available', async () => {
expect(true).toEqual(true); // Skip gracefully
});
return;
}
throw error;
}
});
tap.test('should register Marketstack provider with the service', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
stockService.register(marketstackProvider);
const providers = stockService.getAllProviders();
expect(providers).toContainEqual(marketstackProvider);
expect(stockService.getProvider('Marketstack')).toEqual(marketstackProvider);
});
tap.test('should check provider health', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
const health = await stockService.checkProvidersHealth();
expect(health.get('Marketstack')).toEqual(true);
});
tap.test('should fetch single stock price', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
const price = await stockService.getPrice({ ticker: 'AAPL' });
expect(price).toHaveProperty('ticker');
expect(price).toHaveProperty('price');
expect(price).toHaveProperty('currency');
expect(price).toHaveProperty('change');
expect(price).toHaveProperty('changePercent');
expect(price).toHaveProperty('previousClose');
expect(price).toHaveProperty('timestamp');
expect(price).toHaveProperty('provider');
expect(price).toHaveProperty('marketState');
expect(price.ticker).toEqual('AAPL');
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('Marketstack');
expect(price.timestamp).toBeInstanceOf(Date);
expect(price.marketState).toEqual('CLOSED'); // EOD data
console.log(`✓ Fetched AAPL: $${price.price} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)`);
});
tap.test('should fetch multiple stock prices', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
const prices = await stockService.getPrices({
tickers: testTickers
});
expect(prices).toBeArray();
expect(prices.length).toBeGreaterThan(0);
expect(prices.length).toBeLessThanOrEqual(testTickers.length);
for (const price of prices) {
expect(testTickers).toContain(price.ticker);
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('Marketstack');
expect(price.marketState).toEqual('CLOSED');
console.log(` ${price.ticker}: $${price.price}`);
}
});
tap.test('should serve cached prices on subsequent requests', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
// First request - should hit the API
const firstRequest = await stockService.getPrice({ ticker: 'AAPL' });
// Second request - should be served from cache
const secondRequest = await stockService.getPrice({ ticker: 'AAPL' });
expect(secondRequest.ticker).toEqual(firstRequest.ticker);
expect(secondRequest.price).toEqual(firstRequest.price);
expect(secondRequest.timestamp).toEqual(firstRequest.timestamp);
console.log('✓ Cache working correctly');
});
tap.test('should handle invalid ticker gracefully', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
try {
await stockService.getPrice({ ticker: invalidTicker });
throw new Error('Should have thrown an error for invalid ticker');
} catch (error) {
expect(error.message).toInclude('Failed to fetch');
console.log('✓ Invalid ticker handled correctly');
}
});
tap.test('should support market checking', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
expect(marketstackProvider.supportsMarket('US')).toEqual(true);
expect(marketstackProvider.supportsMarket('UK')).toEqual(true);
expect(marketstackProvider.supportsMarket('DE')).toEqual(true);
expect(marketstackProvider.supportsMarket('JP')).toEqual(true);
expect(marketstackProvider.supportsMarket('INVALID')).toEqual(false);
console.log('✓ Market support check working');
});
tap.test('should validate ticker format', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
expect(marketstackProvider.supportsTicker('AAPL')).toEqual(true);
expect(marketstackProvider.supportsTicker('MSFT')).toEqual(true);
expect(marketstackProvider.supportsTicker('BRK.B')).toEqual(true);
expect(marketstackProvider.supportsTicker('123456789012')).toEqual(false);
expect(marketstackProvider.supportsTicker('invalid@ticker')).toEqual(false);
console.log('✓ Ticker validation working');
});
tap.test('should get provider statistics', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
const stats = stockService.getProviderStats();
const marketstackStats = stats.get('Marketstack');
expect(marketstackStats).not.toEqual(undefined);
expect(marketstackStats.successCount).toBeGreaterThan(0);
expect(marketstackStats.errorCount).toBeGreaterThanOrEqual(0);
console.log(`✓ Provider stats: ${marketstackStats.successCount} successes, ${marketstackStats.errorCount} errors`);
});
tap.test('should test direct provider methods', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🔍 Testing direct provider methods:');
// Test isAvailable
const available = await marketstackProvider.isAvailable();
expect(available).toEqual(true);
console.log(' ✓ isAvailable() returned true');
// Test fetchData for single ticker
const price = await marketstackProvider.fetchData({ type: 'current', ticker: 'MSFT' }) as opendata.IStockPrice;
expect(price.ticker).toEqual('MSFT');
expect(price.provider).toEqual('Marketstack');
expect(price.price).toBeGreaterThan(0);
console.log(` ✓ fetchData (current) for MSFT: $${price.price}`);
// Test fetchData for batch
const prices = await marketstackProvider.fetchData({
type: 'batch',
tickers: ['AAPL', 'GOOGL']
}) as opendata.IStockPrice[];
expect(prices.length).toBeGreaterThan(0);
console.log(` ✓ fetchData (batch) returned ${prices.length} prices`);
for (const p of prices) {
console.log(` ${p.ticker}: $${p.price}`);
}
});
tap.test('should fetch sample EOD data', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n📊 Sample EOD Stock Data from Marketstack:');
console.log('═'.repeat(65));
const sampleTickers = [
{ ticker: 'AAPL', name: 'Apple Inc.' },
{ ticker: 'MSFT', name: 'Microsoft Corp.' },
{ ticker: 'GOOGL', name: 'Alphabet Inc.' },
{ ticker: 'AMZN', name: 'Amazon.com Inc.' },
{ ticker: 'TSLA', name: 'Tesla Inc.' }
];
try {
const prices = await marketstackProvider.fetchData({
type: 'batch',
tickers: sampleTickers.map(t => t.ticker)
}) as opendata.IStockPrice[];
const priceMap = new Map(prices.map(p => [p.ticker, p]));
for (const stock of sampleTickers) {
const price = priceMap.get(stock.ticker);
if (price) {
const changeSymbol = price.change >= 0 ? '↑' : '↓';
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
const resetColor = '\x1b[0m';
console.log(
`${stock.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
minimumFractionDigits: 2,
maximumFractionDigits: 2
}).padStart(10)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
);
}
}
console.log('═'.repeat(65));
console.log(`Provider: Marketstack (EOD Data)`);
console.log(`Last updated: ${new Date().toLocaleString()}\n`);
} catch (error) {
console.log('Error fetching sample data:', error.message);
}
expect(true).toEqual(true);
});
tap.test('should clear cache', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
// Ensure we have something in cache
await stockService.getPrice({ ticker: 'AAPL' });
// Clear cache
stockService.clearCache();
console.log('✓ Cache cleared');
// Next request should hit the API again
const price = await stockService.getPrice({ ticker: 'AAPL' });
expect(price).not.toEqual(undefined);
});
// Phase 1 Feature Tests
tap.test('should fetch data using new unified API (current price)', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🎯 Testing Phase 1: Unified getData API');
const price = await stockService.getData({
type: 'current',
ticker: 'MSFT'
});
expect(price).not.toEqual(undefined);
expect((price as opendata.IStockPrice).ticker).toEqual('MSFT');
expect((price as opendata.IStockPrice).dataType).toEqual('eod');
expect((price as opendata.IStockPrice).fetchedAt).toBeInstanceOf(Date);
console.log(`✓ Fetched current price: $${(price as opendata.IStockPrice).price}`);
});
tap.test('should fetch historical data with date range', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n📅 Testing Phase 1: Historical Data Retrieval');
const fromDate = new Date('2024-12-01');
const toDate = new Date('2024-12-31');
const prices = await stockService.getData({
type: 'historical',
ticker: 'AAPL',
from: fromDate,
to: toDate,
sort: 'DESC'
});
expect(prices).toBeArray();
expect((prices as opendata.IStockPrice[]).length).toBeGreaterThan(0);
console.log(`✓ Fetched ${(prices as opendata.IStockPrice[]).length} historical prices`);
// Verify all data types are 'eod'
for (const price of (prices as opendata.IStockPrice[])) {
expect(price.dataType).toEqual('eod');
expect(price.ticker).toEqual('AAPL');
}
console.log('✓ All prices have correct dataType');
});
tap.test('should include OHLCV data in responses', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n📊 Testing Phase 1: OHLCV Data');
const price = await stockService.getData({
type: 'current',
ticker: 'GOOGL'
});
const stockPrice = price as opendata.IStockPrice;
// Verify OHLCV fields are present
expect(stockPrice.open).not.toEqual(undefined);
expect(stockPrice.high).not.toEqual(undefined);
expect(stockPrice.low).not.toEqual(undefined);
expect(stockPrice.price).not.toEqual(undefined); // close
expect(stockPrice.volume).not.toEqual(undefined);
console.log(`✓ OHLCV Data:`);
console.log(` Open: $${stockPrice.open}`);
console.log(` High: $${stockPrice.high}`);
console.log(` Low: $${stockPrice.low}`);
console.log(` Close: $${stockPrice.price}`);
console.log(` Volume: ${stockPrice.volume?.toLocaleString()}`);
});
tap.test('should support exchange filtering', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🌍 Testing Phase 1: Exchange Filtering');
// Note: This test may fail if the exchange doesn't have data for the ticker
// In production, you'd test with tickers known to exist on specific exchanges
try {
const price = await stockService.getData({
type: 'current',
ticker: 'AAPL',
exchange: 'XNAS' // NASDAQ
});
expect(price).not.toEqual(undefined);
console.log(`✓ Successfully filtered by exchange: ${(price as opendata.IStockPrice).exchange}`);
} catch (error) {
console.log('⚠️ Exchange filtering test inconclusive (may need tier upgrade)');
expect(true).toEqual(true); // Don't fail test
}
});
tap.test('should verify smart caching with historical data', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n💾 Testing Phase 1: Smart Caching');
const fromDate = new Date('2024-11-01');
const toDate = new Date('2024-11-30');
// First request - should hit API
const start1 = Date.now();
const prices1 = await stockService.getData({
type: 'historical',
ticker: 'TSLA',
from: fromDate,
to: toDate
});
const duration1 = Date.now() - start1;
// Second request - should be cached (historical data cached forever)
const start2 = Date.now();
const prices2 = await stockService.getData({
type: 'historical',
ticker: 'TSLA',
from: fromDate,
to: toDate
});
const duration2 = Date.now() - start2;
expect((prices1 as opendata.IStockPrice[]).length).toEqual((prices2 as opendata.IStockPrice[]).length);
expect(duration2).toBeLessThan(duration1); // Cached should be much faster
console.log(`✓ First request: ${duration1}ms (API call)`);
console.log(`✓ Second request: ${duration2}ms (cached)`);
console.log(`✓ Speed improvement: ${Math.round((duration1 / duration2) * 10) / 10}x faster`);
});
// Company Name Feature Tests
tap.test('should include company name in single price request', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🏢 Testing Company Name Feature: Single Request');
const price = await stockService.getPrice({ ticker: 'AAPL' });
expect(price.companyName).not.toEqual(undefined);
expect(typeof price.companyName).toEqual('string');
expect(price.companyName).toInclude('Apple');
console.log(`✓ Company name retrieved: "${price.companyName}"`);
console.log(` Ticker: ${price.ticker}`);
console.log(` Price: $${price.price}`);
console.log(` Company: ${price.companyName}`);
});
tap.test('should include company names in batch price request', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🏢 Testing Company Name Feature: Batch Request');
const prices = await stockService.getPrices({
tickers: ['AAPL', 'MSFT', 'GOOGL']
});
expect(prices).toBeArray();
expect(prices.length).toBeGreaterThan(0);
console.log(`✓ Fetched ${prices.length} prices with company names:`);
for (const price of prices) {
expect(price.companyName).not.toEqual(undefined);
expect(typeof price.companyName).toEqual('string');
console.log(` ${price.ticker.padEnd(6)} - ${price.companyName}`);
}
});
tap.test('should include company name in historical data', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n🏢 Testing Company Name Feature: Historical Data');
const prices = await stockService.getData({
type: 'historical',
ticker: 'TSLA',
from: new Date('2025-10-01'),
to: new Date('2025-10-05')
});
expect(prices).toBeArray();
const historicalPrices = prices as opendata.IStockPrice[];
expect(historicalPrices.length).toBeGreaterThan(0);
// All historical records should have the same company name
for (const price of historicalPrices) {
expect(price.companyName).not.toEqual(undefined);
expect(typeof price.companyName).toEqual('string');
}
const firstPrice = historicalPrices[0];
console.log(`✓ Historical records include company name: "${firstPrice.companyName}"`);
console.log(` Ticker: ${firstPrice.ticker}`);
console.log(` Records: ${historicalPrices.length}`);
console.log(` Date range: ${historicalPrices[historicalPrices.length - 1].timestamp.toISOString().split('T')[0]} to ${firstPrice.timestamp.toISOString().split('T')[0]}`);
});
tap.test('should verify company name is included with zero extra API calls', async () => {
if (!marketstackProvider) {
console.log('⚠️ Skipping - Marketstack provider not initialized');
return;
}
console.log('\n⚡ Testing Company Name Efficiency: Zero Extra API Calls');
// Clear cache to ensure we're making fresh API calls
stockService.clearCache();
// Single request timing
const start1 = Date.now();
const singlePrice = await stockService.getPrice({ ticker: 'AMZN' });
const duration1 = Date.now() - start1;
expect(singlePrice.companyName).not.toEqual(undefined);
// Batch request timing
stockService.clearCache();
const start2 = Date.now();
const batchPrices = await stockService.getPrices({ tickers: ['NVDA', 'AMD', 'INTC'] });
const duration2 = Date.now() - start2;
for (const price of batchPrices) {
expect(price.companyName).not.toEqual(undefined);
}
console.log(`✓ Single request (with company name): ${duration1}ms`);
console.log(`✓ Batch request (with company names): ${duration2}ms`);
console.log(`✓ Company names included in standard EOD response - zero extra calls!`);
console.log(` Single: ${singlePrice.ticker} - "${singlePrice.companyName}"`);
for (const price of batchPrices) {
console.log(` Batch: ${price.ticker} - "${price.companyName}"`);
}
});
export default tap.start();

View File

@@ -0,0 +1,261 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
// Test configuration
const TEST_USER_AGENT = 'fin.cx test@fin.cx';
const TEST_TICKER = 'AAPL'; // Apple Inc - well-known test case
const RATE_LIMIT_DELAY = 150; // 150ms between requests (< 10 req/sec)
tap.test('SEC EDGAR Provider - Constructor', async () => {
await tap.test('should create provider with valid User-Agent', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
expect(provider.name).toEqual('SEC EDGAR');
expect(provider.priority).toEqual(100);
expect(provider.requiresAuth).toBe(false);
expect(provider.rateLimit?.requestsPerMinute).toEqual(600);
});
await tap.test('should throw error if User-Agent is missing', async () => {
expect(() => {
new opendata.SecEdgarProvider({
userAgent: ''
});
}).toThrow('User-Agent is required');
});
await tap.test('should throw error if User-Agent format is invalid', async () => {
expect(() => {
new opendata.SecEdgarProvider({
userAgent: 'InvalidFormat'
});
}).toThrow('Invalid User-Agent format');
});
await tap.test('should accept valid User-Agent with space and email', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: 'MyCompany contact@example.com'
});
expect(provider).toBeInstanceOf(opendata.SecEdgarProvider);
});
});
tap.test('SEC EDGAR Provider - Availability', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
await tap.test('should report as available', async () => {
const isAvailable = await provider.isAvailable();
expect(isAvailable).toBe(true);
});
});
tap.test('SEC EDGAR Provider - Fetch Fundamentals', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT,
timeout: 30000
});
await tap.test('should fetch fundamentals for Apple (AAPL)', async () => {
const fundamentals = await provider.fetchData({
type: 'fundamentals-current',
ticker: TEST_TICKER
});
// Verify structure
expect(fundamentals).toBeDefined();
expect(fundamentals).not.toBeInstanceOf(Array);
const data = fundamentals as opendata.IStockFundamentals;
// Basic fields
expect(data.ticker).toEqual('AAPL');
expect(data.cik).toBeDefined();
expect(data.companyName).toEqual('Apple Inc.');
expect(data.provider).toEqual('SEC EDGAR');
expect(data.timestamp).toBeInstanceOf(Date);
expect(data.fetchedAt).toBeInstanceOf(Date);
// Financial metrics (Apple should have all of these)
expect(data.earningsPerShareDiluted).toBeDefined();
expect(data.earningsPerShareDiluted).toBeGreaterThan(0);
expect(data.sharesOutstanding).toBeDefined();
expect(data.sharesOutstanding).toBeGreaterThan(0);
expect(data.revenue).toBeDefined();
expect(data.revenue).toBeGreaterThan(0);
expect(data.netIncome).toBeDefined();
expect(data.assets).toBeDefined();
expect(data.assets).toBeGreaterThan(0);
expect(data.liabilities).toBeDefined();
expect(data.stockholdersEquity).toBeDefined();
// Metadata
expect(data.fiscalYear).toBeDefined();
console.log('\n📊 Sample Apple Fundamentals:');
console.log(` Company: ${data.companyName} (CIK: ${data.cik})`);
console.log(` EPS (Diluted): $${data.earningsPerShareDiluted?.toFixed(2)}`);
console.log(` Shares Outstanding: ${(data.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
console.log(` Revenue: $${(data.revenue! / 1_000_000_000).toFixed(2)}B`);
console.log(` Net Income: $${(data.netIncome! / 1_000_000_000).toFixed(2)}B`);
console.log(` Assets: $${(data.assets! / 1_000_000_000).toFixed(2)}B`);
console.log(` Fiscal Year: ${data.fiscalYear}`);
});
await tap.test('should throw error for invalid ticker', async () => {
try {
await provider.fetchData({
type: 'fundamentals-current',
ticker: 'INVALIDTICKER123456'
});
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('CIK not found');
}
});
});
tap.test('SEC EDGAR Provider - Batch Fetch', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT,
timeout: 30000
});
await tap.test('should fetch fundamentals for multiple tickers', async () => {
const result = await provider.fetchData({
type: 'fundamentals-batch',
tickers: ['AAPL', 'MSFT']
});
expect(result).toBeInstanceOf(Array);
const fundamentalsList = result as opendata.IStockFundamentals[];
expect(fundamentalsList.length).toEqual(2);
// Check Apple
const apple = fundamentalsList.find(f => f.ticker === 'AAPL');
expect(apple).toBeDefined();
expect(apple!.companyName).toEqual('Apple Inc.');
expect(apple!.earningsPerShareDiluted).toBeGreaterThan(0);
// Check Microsoft
const microsoft = fundamentalsList.find(f => f.ticker === 'MSFT');
expect(microsoft).toBeDefined();
expect(microsoft!.companyName).toContain('Microsoft');
expect(microsoft!.earningsPerShareDiluted).toBeGreaterThan(0);
console.log('\n📊 Batch Fetch Results:');
fundamentalsList.forEach(f => {
console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`);
});
});
});
tap.test('SEC EDGAR Provider - CIK Caching', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
await tap.test('should cache CIK lookups', async () => {
// Clear cache first
provider.clearCache();
let stats = provider.getCacheStats();
expect(stats.cikCacheSize).toEqual(0);
// First fetch (should populate cache)
await provider.fetchData({
type: 'fundamentals-current',
ticker: 'AAPL'
});
stats = provider.getCacheStats();
expect(stats.cikCacheSize).toBeGreaterThan(0);
console.log(`\n💾 CIK Cache: ${stats.cikCacheSize} entries`);
});
await tap.test('should clear cache', async () => {
provider.clearCache();
const stats = provider.getCacheStats();
expect(stats.cikCacheSize).toEqual(0);
expect(stats.hasTickerList).toBe(false);
});
});
tap.test('SEC EDGAR Provider - Rate Limiting', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
await tap.test('should handle multiple rapid requests without exceeding rate limit', async () => {
// Make 5 requests in succession
// Rate limiter should ensure we don't exceed 10 req/sec
const tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'META'];
const startTime = Date.now();
const promises = tickers.map(ticker =>
provider.fetchData({
type: 'fundamentals-current',
ticker
})
);
const results = await Promise.all(promises);
const duration = Date.now() - startTime;
expect(results.length).toEqual(5);
console.log(`\n⏱ 5 requests completed in ${duration}ms (avg: ${Math.round(duration / 5)}ms/request)`);
// Verify all results are valid
results.forEach((result, index) => {
expect(result).toBeDefined();
const data = result as opendata.IStockFundamentals;
expect(data.ticker).toEqual(tickers[index]);
expect(data.earningsPerShareDiluted).toBeGreaterThan(0);
});
});
});
tap.test('SEC EDGAR Provider - Market Cap Calculation', async () => {
const provider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
await tap.test('should provide data needed for market cap calculation', async () => {
const fundamentals = await provider.fetchData({
type: 'fundamentals-current',
ticker: 'AAPL'
}) as opendata.IStockFundamentals;
expect(fundamentals.sharesOutstanding).toBeDefined();
expect(fundamentals.earningsPerShareDiluted).toBeDefined();
// Simulate current price (in real usage, this comes from price provider)
const simulatedPrice = 270.37;
// Calculate market cap
const marketCap = fundamentals.sharesOutstanding! * simulatedPrice;
const pe = simulatedPrice / fundamentals.earningsPerShareDiluted!;
console.log('\n💰 Calculated Metrics (with simulated price $270.37):');
console.log(` Shares Outstanding: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
console.log(` Market Cap: $${(marketCap / 1_000_000_000_000).toFixed(2)}T`);
console.log(` EPS: $${fundamentals.earningsPerShareDiluted!.toFixed(2)}`);
console.log(` P/E Ratio: ${pe.toFixed(2)}`);
expect(marketCap).toBeGreaterThan(0);
expect(pe).toBeGreaterThan(0);
});
});
export default tap.start();

View File

@@ -0,0 +1,365 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
/**
* Test to verify we NEVER return stale intraday data
* Even when cache hasn't expired, we should check for new data
*/
class MockIntradayProvider implements opendata.IStockProvider {
name = 'MockIntradayProvider';
priority = 100;
requiresAuth = false;
public fetchCount = 0;
public lastRequestDate: Date | undefined;
private currentDataCount = 10; // Start with 10 records
private baseTime = new Date('2025-01-07T09:30:00.000Z');
async fetchData(request: opendata.IStockDataRequest): Promise<opendata.IStockPrice | opendata.IStockPrice[]> {
this.fetchCount++;
if (request.type === 'intraday') {
this.lastRequestDate = request.date;
const startTime = request.date || this.baseTime;
const prices: opendata.IStockPrice[] = [];
// Simulate provider returning data AFTER the requested date
for (let i = 0; i < this.currentDataCount; i++) {
const timestamp = new Date(startTime.getTime() + i * 60 * 1000);
// Only return data AFTER request date if date filter is present
if (request.date && timestamp <= request.date) {
continue;
}
prices.push({
ticker: request.ticker,
price: 100 + i,
currency: 'USD',
timestamp,
fetchedAt: new Date(),
provider: this.name,
dataType: 'intraday',
marketState: 'REGULAR',
open: 100,
high: 101,
low: 99,
volume: 1000000,
change: 0,
changePercent: 0,
previousClose: 100
});
}
return prices;
}
throw new Error('Only intraday supported in this mock');
}
async isAvailable(): Promise<boolean> {
return true;
}
public addNewRecords(count: number): void {
this.currentDataCount += count;
}
public advanceTime(minutes: number): void {
this.baseTime = new Date(this.baseTime.getTime() + minutes * 60 * 1000);
}
}
let stockService: opendata.StockPriceService;
let mockProvider: MockIntradayProvider;
tap.test('Stale Data Fix - Setup', async () => {
// Use LONG TTL so cache doesn't expire during test
stockService = new opendata.StockPriceService({
ttl: 300000, // 5 minutes
maxEntries: 1000
});
mockProvider = new MockIntradayProvider();
stockService.register(mockProvider);
console.log('✓ Service initialized with 5-minute cache TTL');
});
tap.test('Stale Data Fix - Check for New Data Even When Cache Valid', async () => {
await tap.test('should return cached data if less than 1 minute old (freshness check)', async () => {
stockService.clearCache();
mockProvider.fetchCount = 0;
mockProvider.currentDataCount = 10;
console.log('\n📊 Scenario: Request twice within 1 minute\n');
// First request - fetch 10 records
console.log('⏰ First request (initial fetch)');
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 1000
});
expect(result1).toBeArray();
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCount).toEqual(1);
const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
console.log(` ✓ Fetched 10 records, latest: ${latestTimestamp1.toISOString()}`);
// Second request immediately - should return cache (data < 1min old)
console.log('\n⏰ Second request (< 1 minute later)');
mockProvider.fetchCount = 0;
mockProvider.addNewRecords(10); // New data available, but won't fetch yet
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'AAPL',
interval: '1min',
limit: 1000
});
// Should return cached data (freshness check prevents fetch)
expect((result2 as opendata.IStockPrice[]).length).toEqual(10);
expect(mockProvider.fetchCount).toEqual(0); // No provider call
console.log(` ✓ Returned cached 10 records (no provider call)`);
console.log(` ✓ Freshness check: Data < 1min old, no fetch needed`);
});
await tap.test('should fetch NEW data when cache is > 1 minute old', async () => {
stockService.clearCache();
mockProvider.fetchCount = 0;
mockProvider.currentDataCount = 10;
console.log('\n📊 Scenario: Request after 2 minutes (data > 1min old)\n');
// First request - fetch 10 records at 9:30am
console.log('⏰ 9:30:00 - First request (initial fetch)');
const result1 = await stockService.getData({
type: 'intraday',
ticker: 'MSFT',
interval: '1min',
limit: 1000
});
expect(result1).toBeArray();
expect((result1 as opendata.IStockPrice[]).length).toEqual(10);
const latestTimestamp1 = (result1 as opendata.IStockPrice[])[9].timestamp;
console.log(` ✓ Fetched 10 records, latest: ${latestTimestamp1.toISOString()}`);
// Advance time by 2 minutes - now data is > 1 minute old
console.log('\n⏰ 9:32:00 - Second request (2 minutes later, data > 1min old)');
console.log(' 📝 Advancing provider time by 2 minutes...');
mockProvider.fetchCount = 0;
mockProvider.advanceTime(2); // Advance 2 minutes
mockProvider.addNewRecords(10); // Now provider has 20 records total
const result2 = await stockService.getData({
type: 'intraday',
ticker: 'MSFT',
interval: '1min',
limit: 1000
});
expect(result2).toBeArray();
const prices2 = result2 as opendata.IStockPrice[];
// Should have 20 records (10 cached + 10 new)
expect(prices2.length).toEqual(20);
// Should have made a provider call (data was stale)
expect(mockProvider.fetchCount).toBeGreaterThan(0);
const latestTimestamp2 = prices2[prices2.length - 1].timestamp;
console.log(` ✓ Now have ${prices2.length} records, latest: ${latestTimestamp2.toISOString()}`);
console.log(` ✓ Provider calls: ${mockProvider.fetchCount} (fetched new data)`);
console.log(` ✓ Data was > 1min old, incremental fetch triggered!`);
// Verify we got NEW data
expect(latestTimestamp2.getTime()).toBeGreaterThan(latestTimestamp1.getTime());
console.log('\n✅ SUCCESS: Fetched new data when cache was stale!');
});
await tap.test('should handle polling with > 1 minute intervals efficiently', async () => {
stockService.clearCache();
mockProvider.fetchCount = 0;
mockProvider.currentDataCount = 100;
console.log('\n📊 Scenario: Dashboard polling every 2 minutes\n');
// Initial request at 9:30am
console.log('⏰ 9:30:00 - Request 1 (initial fetch)');
await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min',
limit: 1000
});
expect(mockProvider.fetchCount).toEqual(1);
console.log(` ✓ Fetched 100 records (provider calls: 1)`);
let totalProviderCalls = 1;
let totalNewRecords = 0;
// Simulate 3 polling refreshes (2 minutes apart, 5 new records each)
for (let i = 2; i <= 4; i++) {
mockProvider.fetchCount = 0;
mockProvider.advanceTime(2); // Advance 2 minutes (triggers freshness check)
mockProvider.addNewRecords(5);
totalNewRecords += 5;
const minutes = (i - 1) * 2;
console.log(`\n⏰ 9:${30 + minutes}:00 - Request ${i} (${minutes} minutes later, +5 new records)`);
const result = await stockService.getData({
type: 'intraday',
ticker: 'GOOGL',
interval: '1min',
limit: 1000
});
const expectedTotal = 100 + totalNewRecords;
expect((result as opendata.IStockPrice[]).length).toEqual(expectedTotal);
// Should have made exactly 1 provider call (incremental fetch)
expect(mockProvider.fetchCount).toEqual(1);
totalProviderCalls++;
console.log(` ✓ Now have ${expectedTotal} records (incremental fetch: 1 call)`);
}
console.log(`\n📊 Summary:`);
console.log(` Total requests: 4`);
console.log(` Total provider calls: ${totalProviderCalls}`);
console.log(` New records fetched: ${totalNewRecords}`);
console.log(` Without incremental cache: Would fetch 100 records × 3 refreshes = 300 records`);
console.log(` With incremental cache: Only fetched ${totalNewRecords} new records`);
console.log(` Data transfer reduction: ${Math.round((1 - (totalNewRecords / 300)) * 100)}%`);
console.log('\n✅ SUCCESS: Only fetched NEW data on each refresh!');
});
});
tap.test('Stale Data Fix - Verify No Regression for Other Request Types', async () => {
await tap.test('historical requests should still use simple cache', async () => {
stockService.clearCache();
// Mock provider that counts calls
let historicalCallCount = 0;
const historicalProvider: opendata.IStockProvider = {
name: 'HistoricalMock',
priority: 100,
requiresAuth: false,
async fetchData() {
historicalCallCount++;
return [{
ticker: 'TEST',
price: 100,
currency: 'USD',
timestamp: new Date('2025-01-01'),
fetchedAt: new Date(),
provider: 'HistoricalMock',
dataType: 'eod',
marketState: 'CLOSED',
open: 99,
high: 101,
low: 98,
volume: 1000000,
change: 1,
changePercent: 1,
previousClose: 99
}];
},
async isAvailable() { return true; }
};
const testService = new opendata.StockPriceService({ ttl: 60000 });
testService.register(historicalProvider);
// First request
await testService.getData({
type: 'historical',
ticker: 'TEST',
from: new Date('2025-01-01'),
to: new Date('2025-01-31')
});
expect(historicalCallCount).toEqual(1);
// Second request - should use cache (not incremental fetch)
await testService.getData({
type: 'historical',
ticker: 'TEST',
from: new Date('2025-01-01'),
to: new Date('2025-01-31')
});
// Should still be 1 (used cache)
expect(historicalCallCount).toEqual(1);
console.log('✓ Historical requests use simple cache (no incremental fetch)');
});
await tap.test('current price requests should still use simple cache', async () => {
stockService.clearCache();
let currentCallCount = 0;
const currentProvider: opendata.IStockProvider = {
name: 'CurrentMock',
priority: 100,
requiresAuth: false,
async fetchData() {
currentCallCount++;
return {
ticker: 'TEST',
price: 150,
currency: 'USD',
timestamp: new Date(),
fetchedAt: new Date(),
provider: 'CurrentMock',
dataType: 'eod',
marketState: 'CLOSED',
open: 149,
high: 151,
low: 148,
volume: 5000000,
change: 1,
changePercent: 0.67,
previousClose: 149
};
},
async isAvailable() { return true; }
};
const testService = new opendata.StockPriceService({ ttl: 60000 });
testService.register(currentProvider);
// First request
await testService.getData({
type: 'current',
ticker: 'TEST'
});
expect(currentCallCount).toEqual(1);
// Second request - should use cache
await testService.getData({
type: 'current',
ticker: 'TEST'
});
expect(currentCallCount).toEqual(1);
console.log('✓ Current price requests use simple cache');
});
});
export default tap.start();

View File

@@ -0,0 +1,418 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
const TEST_USER_AGENT = 'fin.cx test@fin.cx';
tap.test('StockDataService - Basic Setup', async () => {
await tap.test('should create StockDataService instance', async () => {
const service = new opendata.StockDataService({
cache: {
priceTTL: 60000, // 1 minute for testing
fundamentalsTTL: 120000, // 2 minutes for testing
maxEntries: 100
}
});
expect(service).toBeInstanceOf(opendata.StockDataService);
const stats = service.getCacheStats();
expect(stats.priceCache.ttl).toEqual(60000);
expect(stats.fundamentalsCache.ttl).toEqual(120000);
expect(stats.maxEntries).toEqual(100);
});
});
tap.test('StockDataService - Provider Registration', async () => {
const service = new opendata.StockDataService();
await tap.test('should register price provider', async () => {
const yahooProvider = new opendata.YahooFinanceProvider();
service.registerPriceProvider(yahooProvider);
const providers = service.getPriceProviders();
expect(providers.length).toEqual(1);
expect(providers[0].name).toEqual('Yahoo Finance');
});
await tap.test('should register fundamentals provider', async () => {
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerFundamentalsProvider(secProvider);
const providers = service.getFundamentalsProviders();
expect(providers.length).toEqual(1);
expect(providers[0].name).toEqual('SEC EDGAR');
});
await tap.test('should unregister providers', async () => {
service.unregisterPriceProvider('Yahoo Finance');
service.unregisterFundamentalsProvider('SEC EDGAR');
expect(service.getPriceProviders().length).toEqual(0);
expect(service.getFundamentalsProviders().length).toEqual(0);
});
});
tap.test('StockDataService - Price Fetching', async () => {
const service = new opendata.StockDataService();
const yahooProvider = new opendata.YahooFinanceProvider();
service.registerPriceProvider(yahooProvider);
await tap.test('should fetch single price', async () => {
const price = await service.getPrice('AAPL');
expect(price).toBeDefined();
expect(price.ticker).toEqual('AAPL');
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('Yahoo Finance');
expect(price.timestamp).toBeInstanceOf(Date);
console.log(`\n💵 Single Price: ${price.ticker} = $${price.price.toFixed(2)}`);
});
await tap.test('should fetch batch prices', async () => {
const prices = await service.getPrices(['AAPL', 'MSFT', 'GOOGL']);
expect(prices).toBeInstanceOf(Array);
expect(prices.length).toBeGreaterThan(0);
expect(prices.length).toBeLessThanOrEqual(3);
console.log('\n💵 Batch Prices:');
prices.forEach(p => {
console.log(` ${p.ticker}: $${p.price.toFixed(2)}`);
});
});
await tap.test('should cache prices', async () => {
// Clear cache
service.clearCache();
const stats1 = service.getCacheStats();
expect(stats1.priceCache.size).toEqual(0);
// Fetch price (should hit API)
const start1 = Date.now();
await service.getPrice('AAPL');
const duration1 = Date.now() - start1;
const stats2 = service.getCacheStats();
expect(stats2.priceCache.size).toEqual(1);
// Fetch again (should hit cache - much faster)
const start2 = Date.now();
await service.getPrice('AAPL');
const duration2 = Date.now() - start2;
expect(duration2).toBeLessThan(duration1);
console.log('\n⚡ Cache Performance:');
console.log(` First fetch: ${duration1}ms`);
console.log(` Cached fetch: ${duration2}ms`);
console.log(` Speedup: ${Math.round(duration1 / duration2)}x`);
});
});
tap.test('StockDataService - Fundamentals Fetching', async () => {
const service = new opendata.StockDataService();
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerFundamentalsProvider(secProvider);
await tap.test('should fetch single fundamentals', async () => {
const fundamentals = await service.getFundamentals('AAPL');
expect(fundamentals).toBeDefined();
expect(fundamentals.ticker).toEqual('AAPL');
expect(fundamentals.companyName).toEqual('Apple Inc.');
expect(fundamentals.provider).toEqual('SEC EDGAR');
expect(fundamentals.earningsPerShareDiluted).toBeGreaterThan(0);
expect(fundamentals.sharesOutstanding).toBeGreaterThan(0);
console.log('\n📊 Single Fundamentals:');
console.log(` ${fundamentals.ticker}: ${fundamentals.companyName}`);
console.log(` EPS: $${fundamentals.earningsPerShareDiluted?.toFixed(2)}`);
console.log(` Shares: ${(fundamentals.sharesOutstanding! / 1_000_000_000).toFixed(2)}B`);
});
await tap.test('should fetch batch fundamentals', async () => {
const fundamentals = await service.getBatchFundamentals(['AAPL', 'MSFT']);
expect(fundamentals).toBeInstanceOf(Array);
expect(fundamentals.length).toEqual(2);
console.log('\n📊 Batch Fundamentals:');
fundamentals.forEach(f => {
console.log(` ${f.ticker}: ${f.companyName} - EPS: $${f.earningsPerShareDiluted?.toFixed(2)}`);
});
});
await tap.test('should cache fundamentals', async () => {
// Clear cache
service.clearCache();
const stats1 = service.getCacheStats();
expect(stats1.fundamentalsCache.size).toEqual(0);
// Fetch fundamentals (should hit API)
await service.getFundamentals('AAPL');
const stats2 = service.getCacheStats();
expect(stats2.fundamentalsCache.size).toEqual(1);
});
});
tap.test('StockDataService - Complete Stock Data', async () => {
const service = new opendata.StockDataService();
// Register both providers
const yahooProvider = new opendata.YahooFinanceProvider();
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerPriceProvider(yahooProvider);
service.registerFundamentalsProvider(secProvider);
await tap.test('should fetch complete stock data with string', async () => {
const data = await service.getStockData('AAPL');
expect(data).toBeDefined();
expect(data.ticker).toEqual('AAPL');
expect(data.price).toBeDefined();
expect(data.price.ticker).toEqual('AAPL');
expect(data.fundamentals).toBeDefined();
expect(data.fundamentals?.ticker).toEqual('AAPL');
expect(data.fetchedAt).toBeInstanceOf(Date);
// Check automatic enrichment
expect(data.fundamentals?.marketCap).toBeDefined();
expect(data.fundamentals?.priceToEarnings).toBeDefined();
expect(data.fundamentals?.marketCap).toBeGreaterThan(0);
expect(data.fundamentals?.priceToEarnings).toBeGreaterThan(0);
console.log('\n✨ Complete Stock Data (Auto-Enriched):');
console.log(` ${data.ticker}: ${data.fundamentals?.companyName}`);
console.log(` Price: $${data.price.price.toFixed(2)}`);
console.log(` Market Cap: $${(data.fundamentals!.marketCap! / 1_000_000_000_000).toFixed(2)}T`);
console.log(` P/E Ratio: ${data.fundamentals!.priceToEarnings!.toFixed(2)}`);
});
await tap.test('should fetch complete stock data with request object', async () => {
const data = await service.getStockData({
ticker: 'MSFT',
includeFundamentals: true,
enrichFundamentals: true
});
expect(data).toBeDefined();
expect(data.ticker).toEqual('MSFT');
expect(data.price).toBeDefined();
expect(data.fundamentals).toBeDefined();
expect(data.fundamentals?.marketCap).toBeDefined();
expect(data.fundamentals?.priceToEarnings).toBeDefined();
});
await tap.test('should fetch complete stock data without fundamentals', async () => {
const data = await service.getStockData({
ticker: 'GOOGL',
includeFundamentals: false
});
expect(data).toBeDefined();
expect(data.ticker).toEqual('GOOGL');
expect(data.price).toBeDefined();
expect(data.fundamentals).toBeUndefined();
});
await tap.test('should handle fundamentals fetch failure gracefully', async () => {
// Try a ticker that might not have fundamentals
const data = await service.getStockData({
ticker: 'BTC-USD', // Crypto - no SEC filings
includeFundamentals: true
});
expect(data).toBeDefined();
expect(data.price).toBeDefined();
// Fundamentals might be undefined due to error
console.log(`\n⚠ ${data.ticker} - Price available, Fundamentals: ${data.fundamentals ? 'Yes' : 'No'}`);
});
});
tap.test('StockDataService - Batch Complete Stock Data', async () => {
const service = new opendata.StockDataService();
const yahooProvider = new opendata.YahooFinanceProvider();
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerPriceProvider(yahooProvider);
service.registerFundamentalsProvider(secProvider);
await tap.test('should fetch batch complete data with array', async () => {
const data = await service.getBatchStockData(['AAPL', 'MSFT']);
expect(data).toBeInstanceOf(Array);
expect(data.length).toEqual(2);
data.forEach(stock => {
expect(stock.ticker).toBeDefined();
expect(stock.price).toBeDefined();
expect(stock.fundamentals).toBeDefined();
expect(stock.fundamentals?.marketCap).toBeGreaterThan(0);
expect(stock.fundamentals?.priceToEarnings).toBeGreaterThan(0);
});
console.log('\n✨ Batch Complete Data:');
data.forEach(stock => {
console.log(` ${stock.ticker}: Price $${stock.price.price.toFixed(2)}, P/E ${stock.fundamentals!.priceToEarnings!.toFixed(2)}`);
});
});
await tap.test('should fetch batch complete data with request object', async () => {
const data = await service.getBatchStockData({
tickers: ['AAPL', 'GOOGL'],
includeFundamentals: true,
enrichFundamentals: true
});
expect(data).toBeInstanceOf(Array);
expect(data.length).toEqual(2);
data.forEach(stock => {
expect(stock.fundamentals?.marketCap).toBeGreaterThan(0);
});
});
await tap.test('should fetch batch without enrichment', async () => {
const data = await service.getBatchStockData({
tickers: ['AAPL', 'MSFT'],
includeFundamentals: true,
enrichFundamentals: false
});
expect(data).toBeInstanceOf(Array);
// Check that fundamentals exist but enrichment might not be complete
data.forEach(stock => {
if (stock.fundamentals) {
expect(stock.fundamentals.ticker).toBeDefined();
expect(stock.fundamentals.companyName).toBeDefined();
}
});
});
});
tap.test('StockDataService - Health & Statistics', async () => {
const service = new opendata.StockDataService();
const yahooProvider = new opendata.YahooFinanceProvider();
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerPriceProvider(yahooProvider);
service.registerFundamentalsProvider(secProvider);
await tap.test('should check providers health', async () => {
const health = await service.checkProvidersHealth();
expect(health.size).toEqual(2);
expect(health.get('Yahoo Finance (price)')).toBe(true);
expect(health.get('SEC EDGAR (fundamentals)')).toBe(true);
console.log('\n💚 Provider Health:');
health.forEach((isHealthy, name) => {
console.log(` ${name}: ${isHealthy ? '✅ Healthy' : '❌ Unhealthy'}`);
});
});
await tap.test('should track provider statistics', async () => {
// Make some requests to generate stats
await service.getPrice('AAPL');
await service.getFundamentals('AAPL');
const stats = service.getProviderStats();
expect(stats.size).toEqual(2);
const yahooStats = stats.get('Yahoo Finance');
expect(yahooStats).toBeDefined();
expect(yahooStats!.type).toEqual('price');
expect(yahooStats!.successCount).toBeGreaterThan(0);
const secStats = stats.get('SEC EDGAR');
expect(secStats).toBeDefined();
expect(secStats!.type).toEqual('fundamentals');
expect(secStats!.successCount).toBeGreaterThan(0);
console.log('\n📈 Provider Statistics:');
stats.forEach((stat, name) => {
console.log(` ${name} (${stat.type}): Success=${stat.successCount}, Errors=${stat.errorCount}`);
});
});
await tap.test('should clear all caches', async () => {
service.clearCache();
const stats = service.getCacheStats();
expect(stats.priceCache.size).toEqual(0);
expect(stats.fundamentalsCache.size).toEqual(0);
});
});
tap.test('StockDataService - Error Handling', async () => {
await tap.test('should throw error when no price provider available', async () => {
const service = new opendata.StockDataService();
try {
await service.getPrice('AAPL');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('No price providers available');
}
});
await tap.test('should throw error when no fundamentals provider available', async () => {
const service = new opendata.StockDataService();
try {
await service.getFundamentals('AAPL');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('No fundamentals providers available');
}
});
await tap.test('should handle invalid ticker for price', async () => {
const service = new opendata.StockDataService();
const yahooProvider = new opendata.YahooFinanceProvider();
service.registerPriceProvider(yahooProvider);
try {
await service.getPrice('INVALIDTICKER123456');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('Failed to fetch price');
}
});
await tap.test('should handle invalid ticker for fundamentals', async () => {
const service = new opendata.StockDataService();
const secProvider = new opendata.SecEdgarProvider({
userAgent: TEST_USER_AGENT
});
service.registerFundamentalsProvider(secProvider);
try {
await service.getFundamentals('INVALIDTICKER123456');
throw new Error('Should have thrown error');
} catch (error) {
expect(error.message).toContain('CIK not found');
}
});
});
export default tap.start();

280
test/test.stocks.ts Normal file
View File

@@ -0,0 +1,280 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js';
// Test data
const testTickers = ['AAPL', 'MSFT', 'GOOGL'];
const invalidTicker = 'INVALID_TICKER_XYZ';
let stockService: opendata.StockPriceService;
let yahooProvider: opendata.YahooFinanceProvider;
tap.test('should create StockPriceService instance', async () => {
stockService = new opendata.StockPriceService({
ttl: 30000, // 30 seconds cache
maxEntries: 100
});
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
});
tap.test('should create YahooFinanceProvider instance', async () => {
yahooProvider = new opendata.YahooFinanceProvider({
enabled: true,
timeout: 10000,
retryAttempts: 2,
retryDelay: 500
});
expect(yahooProvider).toBeInstanceOf(opendata.YahooFinanceProvider);
expect(yahooProvider.name).toEqual('Yahoo Finance');
expect(yahooProvider.requiresAuth).toEqual(false);
});
tap.test('should register Yahoo provider with the service', async () => {
stockService.register(yahooProvider);
const providers = stockService.getAllProviders();
expect(providers).toContainEqual(yahooProvider);
expect(stockService.getProvider('Yahoo Finance')).toEqual(yahooProvider);
});
tap.test('should check provider health', async () => {
const health = await stockService.checkProvidersHealth();
expect(health.get('Yahoo Finance')).toEqual(true);
});
tap.test('should fetch single stock price', async () => {
const price = await stockService.getPrice({ ticker: 'AAPL' });
expect(price).toHaveProperty('ticker');
expect(price).toHaveProperty('price');
expect(price).toHaveProperty('currency');
expect(price).toHaveProperty('change');
expect(price).toHaveProperty('changePercent');
expect(price).toHaveProperty('previousClose');
expect(price).toHaveProperty('timestamp');
expect(price).toHaveProperty('provider');
expect(price).toHaveProperty('marketState');
expect(price.ticker).toEqual('AAPL');
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('Yahoo Finance');
expect(price.timestamp).toBeInstanceOf(Date);
});
tap.test('should fetch multiple stock prices', async () => {
const prices = await stockService.getPrices({
tickers: testTickers
});
expect(prices).toBeArray();
expect(prices.length).toBeGreaterThan(0);
expect(prices.length).toBeLessThanOrEqual(testTickers.length);
for (const price of prices) {
expect(testTickers).toContain(price.ticker);
expect(price.price).toBeGreaterThan(0);
expect(price.provider).toEqual('Yahoo Finance');
}
});
tap.test('should serve cached prices on subsequent requests', async () => {
// First request - should hit the API
const firstRequest = await stockService.getPrice({ ticker: 'AAPL' });
// Second request - should be served from cache
const secondRequest = await stockService.getPrice({ ticker: 'AAPL' });
expect(secondRequest.ticker).toEqual(firstRequest.ticker);
expect(secondRequest.price).toEqual(firstRequest.price);
expect(secondRequest.timestamp).toEqual(firstRequest.timestamp);
});
tap.test('should handle invalid ticker gracefully', async () => {
try {
await stockService.getPrice({ ticker: invalidTicker });
throw new Error('Should have thrown an error for invalid ticker');
} catch (error) {
expect(error.message).toInclude('Failed to fetch price');
expect(error.message).toInclude(invalidTicker);
}
});
tap.test('should support market checking', async () => {
expect(yahooProvider.supportsMarket('US')).toEqual(true);
expect(yahooProvider.supportsMarket('UK')).toEqual(true);
expect(yahooProvider.supportsMarket('DE')).toEqual(true);
expect(yahooProvider.supportsMarket('INVALID')).toEqual(false);
});
tap.test('should validate ticker format', async () => {
expect(yahooProvider.supportsTicker('AAPL')).toEqual(true);
expect(yahooProvider.supportsTicker('MSFT')).toEqual(true);
expect(yahooProvider.supportsTicker('BRK.B')).toEqual(true);
expect(yahooProvider.supportsTicker('123456789012')).toEqual(false);
expect(yahooProvider.supportsTicker('invalid@ticker')).toEqual(false);
});
tap.test('should get provider statistics', async () => {
const stats = stockService.getProviderStats();
const yahooStats = stats.get('Yahoo Finance');
expect(yahooStats).not.toEqual(undefined);
expect(yahooStats.successCount).toBeGreaterThan(0);
expect(yahooStats.errorCount).toBeGreaterThanOrEqual(0);
});
tap.test('should clear cache', async () => {
// Ensure we have something in cache
await stockService.getPrice({ ticker: 'AAPL' });
// Clear cache
stockService.clearCache();
// Next request should hit the API again (we can't directly test this,
// but we can verify the method doesn't throw)
const price = await stockService.getPrice({ ticker: 'AAPL' });
expect(price).not.toEqual(undefined);
});
tap.test('should handle provider unavailability', async () => {
// Clear cache first to ensure we don't get cached results
stockService.clearCache();
// Unregister all providers
stockService.unregister('Yahoo Finance');
try {
// Use a different ticker to avoid any caching
await stockService.getPrice({ ticker: 'TSLA' });
throw new Error('Should have thrown an error with no providers');
} catch (error: any) {
expect(error.message).toEqual('No stock price providers available');
}
});
tap.test('should fetch major market indicators', async () => {
// Re-register provider if needed
if (!stockService.getProvider('Yahoo Finance')) {
stockService.register(yahooProvider);
}
const marketIndicators = [
// Indices
{ ticker: '^GSPC', name: 'S&P 500' },
{ ticker: '^IXIC', name: 'NASDAQ' },
{ ticker: '^DJI', name: 'DOW Jones' },
// Tech Stocks
{ ticker: 'AAPL', name: 'Apple' },
{ ticker: 'AMZN', name: 'Amazon' },
{ ticker: 'GOOGL', name: 'Google' },
{ ticker: 'META', name: 'Meta' },
{ ticker: 'MSFT', name: 'Microsoft' },
{ ticker: 'PLTR', name: 'Palantir' },
// Crypto
{ ticker: 'BTC-USD', name: 'Bitcoin' },
{ ticker: 'ETH-USD', name: 'Ethereum' },
{ ticker: 'ADA-USD', name: 'Cardano' },
// Forex & Commodities
{ ticker: 'EURUSD=X', name: 'EUR/USD' },
{ ticker: 'GC=F', name: 'Gold Futures' },
{ ticker: 'CL=F', name: 'Crude Oil Futures' }
];
console.log('\n📊 Current Market Values:');
console.log('═'.repeat(65));
// Fetch all prices in batch for better performance
try {
const prices = await stockService.getPrices({
tickers: marketIndicators.map(i => i.ticker)
});
// Create a map for easy lookup
const priceMap = new Map(prices.map(p => [p.ticker, p]));
// Check which tickers are missing and fetch them individually
const missingTickers: typeof marketIndicators = [];
for (const indicator of marketIndicators) {
if (!priceMap.has(indicator.ticker)) {
missingTickers.push(indicator);
}
}
// Fetch missing tickers individually
if (missingTickers.length > 0) {
for (const indicator of missingTickers) {
try {
const price = await stockService.getPrice({ ticker: indicator.ticker });
priceMap.set(indicator.ticker, price);
} catch (error) {
// Ignore individual errors
}
}
}
// Display all results with section headers
let lastSection = '';
for (const indicator of marketIndicators) {
// Add section headers
if (indicator.ticker.startsWith('^') && lastSection !== 'indices') {
console.log('\n📈 Market Indices');
console.log('─'.repeat(65));
lastSection = 'indices';
} else if (['AAPL', 'AMZN', 'GOOGL', 'META', 'MSFT', 'PLTR'].includes(indicator.ticker) && lastSection !== 'stocks') {
console.log('\n💻 Tech Stocks');
console.log('─'.repeat(65));
lastSection = 'stocks';
} else if (indicator.ticker.includes('-USD') && lastSection !== 'crypto') {
console.log('\n🪙 Cryptocurrencies');
console.log('─'.repeat(65));
lastSection = 'crypto';
} else if ((indicator.ticker.includes('=') || indicator.ticker.includes('=F')) && lastSection !== 'forex') {
console.log('\n💱 Forex & Commodities');
console.log('─'.repeat(65));
lastSection = 'forex';
}
const price = priceMap.get(indicator.ticker);
if (price) {
const changeSymbol = price.change >= 0 ? '↑' : '↓';
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m'; // Green or Red
const resetColor = '\x1b[0m';
console.log(
`${indicator.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
minimumFractionDigits: 2,
maximumFractionDigits: indicator.name.includes('coin') || indicator.name.includes('EUR') || indicator.name === 'Cardano' ? 4 : 2
}).padStart(12)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
);
} else {
console.log(`${indicator.name.padEnd(20)} Data not available`);
}
}
} catch (error) {
console.log('Error fetching market data:', error);
// Fallback to individual fetches
for (const indicator of marketIndicators) {
try {
const price = await stockService.getPrice({ ticker: indicator.ticker });
const changeSymbol = price.change >= 0 ? '↑' : '↓';
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
const resetColor = '\x1b[0m';
console.log(
`${indicator.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
minimumFractionDigits: 2,
maximumFractionDigits: indicator.name.includes('coin') || indicator.name.includes('EUR') || indicator.name === 'Cardano' ? 4 : 2
}).padStart(12)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
);
} catch (error) {
console.log(`${indicator.name.padEnd(20)} Error fetching data`);
}
}
}
console.log('═'.repeat(65));
console.log(`Last updated: ${new Date().toLocaleString()}\n`);
// Test passes if we successfully fetch at least some indicators
expect(true).toEqual(true);
});
export default tap.start();

View File

@@ -1,15 +1,23 @@
import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
import { tapNodeTools } from '@push.rocks/tapbundle/node';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as opendata from '../ts/index.js'
import * as paths from '../ts/paths.js';
import * as plugins from '../ts/plugins.js';
import { BusinessRecord } from '../ts/classes.businessrecord.js';
// Test configuration - explicit paths required
const testNogitDir = plugins.path.join(paths.packageDir, '.nogit');
const testDownloadDir = plugins.path.join(testNogitDir, 'downloads');
const testGermanBusinessDataDir = plugins.path.join(testNogitDir, 'germanbusinessdata');
let testOpenDataInstance: opendata.OpenData;
tap.test('first test', async () => {
testOpenDataInstance = new opendata.OpenData();
testOpenDataInstance = new opendata.OpenData({
nogitDir: testNogitDir,
downloadDir: testDownloadDir,
germanBusinessDataDir: testGermanBusinessDataDir
});
expect(testOpenDataInstance).toBeInstanceOf(opendata.OpenData);
});
@@ -17,30 +25,10 @@ tap.test('should start the instance', async () => {
await testOpenDataInstance.start();
})
tap.skip.test('should build initial data', async () => {
tap.test('should build initial data', async () => {
await testOpenDataInstance.buildInitialDb();
});
const resultsSearch = tap.test('should get the data for a company', async () => {
const result = await testOpenDataInstance.handelsregister.searchCompany('Volkswagen', 20);
console.log(result);
return result;
});
tap.test('should get the data for a specific company', async () => {
let testCompany: BusinessRecord['data']['germanParsedRegistration'] = (await resultsSearch.testResultPromise)[7]['germanParsedRegistration'];
console.log(`trying to find specific company with:`);
console.log(testCompany);
const result = await testOpenDataInstance.handelsregister.getSpecificCompany(testCompany);
console.log(result);
result.files.map(async (file) => {
await file.writeToDir('./.nogit/testoutput');
});
});
tap.test('should stop the instance', async () => {
await testOpenDataInstance.stop();
});

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@fin.cx/opendata',
version: '1.4.3',
description: 'A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.'
version: '3.5.0',
description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
}

View File

@@ -1,7 +1,6 @@
import type { BusinessRecord } from './classes.businessrecord.js';
import type { OpenData } from './classes.main.opendata.js';
import * as plugins from './plugins.js';
import * as paths from './paths.js';
/**
* the HandlesRegister exposed as a class
@@ -9,13 +8,16 @@ import * as paths from './paths.js';
export class HandelsRegister {
private openDataRef: OpenData;
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple());
private downloadDir: string;
private uniqueDowloadFolder: string;
// Puppeteer wrapper instance
public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
constructor(openDataRef: OpenData) {
constructor(openDataRef: OpenData, downloadDirArg: string) {
this.openDataRef = openDataRef;
this.downloadDir = downloadDirArg;
this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple());
}
public async start() {
@@ -76,7 +78,7 @@ export class HandelsRegister {
timeout: 30000,
})
.catch(async (err) => {
await pageArg.screenshot({ path: paths.downloadDir + '/error.png' });
await pageArg.screenshot({ path: `${this.downloadDir}/error.png` as `${string}.png` });
throw err;
});

View File

@@ -1,5 +1,4 @@
import * as plugins from './plugins.js';
import * as paths from './paths.js';
import type { OpenData } from './classes.main.opendata.js';
export type SeedEntryType = {
@@ -41,8 +40,11 @@ export type SeedEntryType = {
};
export class JsonlDataProcessor<T> {
private germanBusinessDataDir: string;
public forEachFunction: (entryArg: T) => Promise<void>;
constructor(forEachFunctionArg: typeof this.forEachFunction) {
constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) {
this.germanBusinessDataDir = germanBusinessDataDirArg;
this.forEachFunction = forEachFunctionArg;
}
@@ -51,9 +53,9 @@ export class JsonlDataProcessor<T> {
dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
) {
const done = plugins.smartpromise.defer();
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir);
if (!dataExists) {
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir);
} else {
}
@@ -96,8 +98,9 @@ export class JsonlDataProcessor<T> {
},
finalFunction: async (streamToolsArg) => {
console.log(`finished processing ${totalRecordsCounter} records.`);
if (!nextRest) return;
if (nextRest) {
JSON.parse(nextRest);
};
done.resolve();
},
})

View File

@@ -4,16 +4,39 @@ import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
import * as paths from './paths.js';
import * as plugins from './plugins.js';
export interface IOpenDataConfig {
downloadDir: string;
germanBusinessDataDir: string;
nogitDir: string;
}
export class OpenData {
public db: plugins.smartdata.SmartdataDb;
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
private serviceQenv: plugins.qenv.Qenv;
private config: IOpenDataConfig;
public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
public handelsregister: HandelsRegister;
public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
constructor(configArg: IOpenDataConfig) {
if (!configArg) {
throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.');
}
if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) {
throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
}
this.config = configArg;
this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir);
}
public async start() {
// Ensure configured directories exist
await plugins.smartfile.fs.ensureDir(this.config.nogitDir);
await plugins.smartfile.fs.ensureDir(this.config.downloadDir);
await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir);
this.db = new plugins.smartdata.SmartdataDb({
mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
@@ -21,7 +44,9 @@ export class OpenData {
mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
});
await this.db.init();
this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => {
this.jsonLDataProcessor = new JsonlDataProcessor(
this.config.germanBusinessDataDir,
async (entryArg) => {
const businessRecord = new this.CBusinessRecord();
businessRecord.id = await this.CBusinessRecord.getNewId();
businessRecord.data.name = entryArg.name;
@@ -31,8 +56,9 @@ export class OpenData {
type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
};
await businessRecord.save();
});
this.handelsregister = new HandelsRegister(this);
}
);
this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
await this.handelsregister.start();
}

View File

@@ -1 +1,2 @@
export * from './classes.main.opendata.js';
export * from './stocks/index.js';

View File

@@ -4,12 +4,3 @@ export const packageDir = plugins.path.join(
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
'../'
);
export const nogitDir = plugins.path.join(packageDir, './.nogit/');
plugins.smartfile.fs.ensureDirSync(nogitDir);
export const downloadDir = plugins.path.join(nogitDir, 'downloads');
plugins.smartfile.fs.ensureDirSync(downloadDir);
export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');

View File

@@ -1,5 +1,5 @@
// node native scope
import * as path from 'path';
import * as path from 'node:path';
export {
path,
@@ -14,6 +14,7 @@ import * as smartbrowser from '@push.rocks/smartbrowser';
import * as smartdata from '@push.rocks/smartdata';
import * as smartdelay from '@push.rocks/smartdelay';
import * as smartfile from '@push.rocks/smartfile';
import * as smartlog from '@push.rocks/smartlog';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
@@ -30,6 +31,7 @@ export {
smartdata,
smartdelay,
smartfile,
smartlog,
smartpath,
smartpromise,
smartrequest,

View File

@@ -0,0 +1,296 @@
import * as plugins from '../plugins.js';
/**
* Base provider entry for tracking provider state
*/
export interface IBaseProviderEntry<TProvider> {
provider: TProvider;
config: IBaseProviderConfig;
lastError?: Error;
lastErrorTime?: Date;
successCount: number;
errorCount: number;
}
/**
* Base provider configuration
*/
export interface IBaseProviderConfig {
enabled: boolean;
priority: number;
timeout?: number;
retryAttempts?: number;
retryDelay?: number;
cacheTTL?: number;
}
/**
* Base provider interface
*/
export interface IBaseProvider {
name: string;
priority: number;
isAvailable(): Promise<boolean>;
readonly requiresAuth: boolean;
readonly rateLimit?: {
requestsPerMinute: number;
requestsPerDay?: number;
};
}
/**
* Cache entry for any data type
*/
export interface IBaseCacheEntry<TData> {
data: TData;
timestamp: Date;
ttl: number;
}
/**
* Base service for managing data providers with caching
* Shared logic extracted from StockPriceService and FundamentalsService
*/
export abstract class BaseProviderService<TProvider extends IBaseProvider, TData> {
protected providers = new Map<string, IBaseProviderEntry<TProvider>>();
protected cache = new Map<string, IBaseCacheEntry<TData>>();
protected logger = console;
protected cacheConfig = {
ttl: 60000, // Default 60 seconds
maxEntries: 10000
};
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
if (cacheConfig) {
this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
}
}
/**
* Register a provider
*/
public register(provider: TProvider, config?: Partial<IBaseProviderConfig>): void {
const defaultConfig: IBaseProviderConfig = {
enabled: true,
priority: provider.priority,
timeout: 30000,
retryAttempts: 2,
retryDelay: 1000,
cacheTTL: this.cacheConfig.ttl
};
const mergedConfig = { ...defaultConfig, ...config };
this.providers.set(provider.name, {
provider,
config: mergedConfig,
successCount: 0,
errorCount: 0
});
console.log(`Registered provider: ${provider.name}`);
}
/**
* Unregister a provider
*/
public unregister(providerName: string): void {
this.providers.delete(providerName);
console.log(`Unregistered provider: ${providerName}`);
}
/**
* Get a specific provider by name
*/
public getProvider(name: string): TProvider | undefined {
return this.providers.get(name)?.provider;
}
/**
* Get all registered providers
*/
public getAllProviders(): TProvider[] {
return Array.from(this.providers.values()).map(entry => entry.provider);
}
/**
* Get enabled providers sorted by priority
*/
public getEnabledProviders(): TProvider[] {
return Array.from(this.providers.values())
.filter(entry => entry.config.enabled)
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
.map(entry => entry.provider);
}
/**
* Check health of all providers
*/
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
const health = new Map<string, boolean>();
for (const [name, entry] of this.providers) {
if (!entry.config.enabled) {
health.set(name, false);
continue;
}
try {
const isAvailable = await entry.provider.isAvailable();
health.set(name, isAvailable);
} catch (error) {
health.set(name, false);
console.error(`Health check failed for ${name}:`, error);
}
}
return health;
}
/**
* Get provider statistics
*/
public getProviderStats(): Map<
string,
{
successCount: number;
errorCount: number;
lastError?: string;
lastErrorTime?: Date;
}
> {
const stats = new Map();
for (const [name, entry] of this.providers) {
stats.set(name, {
successCount: entry.successCount,
errorCount: entry.errorCount,
lastError: entry.lastError?.message,
lastErrorTime: entry.lastErrorTime
});
}
return stats;
}
/**
* Clear all cached data
*/
public clearCache(): void {
this.cache.clear();
console.log('Cache cleared');
}
/**
* Set cache TTL
*/
public setCacheTTL(ttl: number): void {
this.cacheConfig.ttl = ttl;
console.log(`Cache TTL set to ${ttl}ms`);
}
/**
* Get cache statistics
*/
public getCacheStats(): {
size: number;
maxEntries: number;
ttl: number;
} {
return {
size: this.cache.size,
maxEntries: this.cacheConfig.maxEntries,
ttl: this.cacheConfig.ttl
};
}
/**
* Fetch with retry logic
*/
protected async fetchWithRetry<T>(
fetchFn: () => Promise<T>,
config: IBaseProviderConfig
): Promise<T> {
const maxAttempts = config.retryAttempts || 1;
let lastError: Error | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await fetchFn();
} catch (error) {
lastError = error as Error;
if (attempt < maxAttempts) {
const delay = (config.retryDelay || 1000) * attempt;
console.log(`Retry attempt ${attempt} after ${delay}ms`);
await plugins.smartdelay.delayFor(delay);
}
}
}
throw lastError || new Error('Unknown error during fetch');
}
/**
* Get from cache if not expired
*/
protected getFromCache(key: string): TData | null {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
// Check if cache entry has expired
const age = Date.now() - entry.timestamp.getTime();
if (entry.ttl !== Infinity && age > entry.ttl) {
this.cache.delete(key);
return null;
}
return entry.data;
}
/**
* Add to cache with TTL
*/
protected addToCache(key: string, data: TData, ttl?: number): void {
// Enforce max entries limit
if (this.cache.size >= this.cacheConfig.maxEntries) {
// Remove oldest entry
const oldestKey = this.cache.keys().next().value;
if (oldestKey) {
this.cache.delete(oldestKey);
}
}
this.cache.set(key, {
data,
timestamp: new Date(),
ttl: ttl || this.cacheConfig.ttl
});
}
/**
* Track successful fetch for provider
*/
protected trackSuccess(providerName: string): void {
const entry = this.providers.get(providerName);
if (entry) {
entry.successCount++;
}
}
/**
* Track failed fetch for provider
*/
protected trackError(providerName: string, error: Error): void {
const entry = this.providers.get(providerName);
if (entry) {
entry.errorCount++;
entry.lastError = error;
entry.lastErrorTime = new Date();
}
}
}

View File

@@ -0,0 +1,404 @@
import * as plugins from '../plugins.js';
import type {
IFundamentalsProvider,
IFundamentalsProviderConfig,
IFundamentalsProviderRegistry,
IStockFundamentals,
IFundamentalsRequest
} from './interfaces/fundamentals.js';
interface IProviderEntry {
provider: IFundamentalsProvider;
config: IFundamentalsProviderConfig;
lastError?: Error;
lastErrorTime?: Date;
successCount: number;
errorCount: number;
}
interface ICacheEntry {
fundamentals: IStockFundamentals | IStockFundamentals[];
timestamp: Date;
ttl: number;
}
/**
* Service for managing fundamental data providers and caching
* Parallel to StockPriceService but for fundamental data instead of prices
*/
export class FundamentalsService implements IFundamentalsProviderRegistry {
private providers = new Map<string, IProviderEntry>();
private cache = new Map<string, ICacheEntry>();
private logger = console;
private cacheConfig = {
ttl: 90 * 24 * 60 * 60 * 1000, // 90 days default (fundamentals change quarterly)
maxEntries: 10000
};
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
if (cacheConfig) {
this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
}
}
/**
* Register a fundamentals provider
*/
public register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void {
const defaultConfig: IFundamentalsProviderConfig = {
enabled: true,
priority: provider.priority,
timeout: 30000, // Longer timeout for fundamental data
retryAttempts: 2,
retryDelay: 1000,
cacheTTL: this.cacheConfig.ttl
};
const mergedConfig = { ...defaultConfig, ...config };
this.providers.set(provider.name, {
provider,
config: mergedConfig,
successCount: 0,
errorCount: 0
});
console.log(`Registered fundamentals provider: ${provider.name}`);
}
/**
* Unregister a provider
*/
public unregister(providerName: string): void {
this.providers.delete(providerName);
console.log(`Unregistered fundamentals provider: ${providerName}`);
}
/**
* Get a specific provider by name
*/
public getProvider(name: string): IFundamentalsProvider | undefined {
return this.providers.get(name)?.provider;
}
/**
* Get all registered providers
*/
public getAllProviders(): IFundamentalsProvider[] {
return Array.from(this.providers.values()).map(entry => entry.provider);
}
/**
* Get enabled providers sorted by priority
*/
public getEnabledProviders(): IFundamentalsProvider[] {
return Array.from(this.providers.values())
.filter(entry => entry.config.enabled)
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
.map(entry => entry.provider);
}
/**
* Get fundamental data for a single ticker
*/
public async getFundamentals(ticker: string): Promise<IStockFundamentals> {
const result = await this.getData({
type: 'fundamentals-current',
ticker
});
return result as IStockFundamentals;
}
/**
* Get fundamental data for multiple tickers
*/
public async getBatchFundamentals(tickers: string[]): Promise<IStockFundamentals[]> {
const result = await this.getData({
type: 'fundamentals-batch',
tickers
});
return result as IStockFundamentals[];
}
/**
* Unified data fetching method
*/
public async getData(
request: IFundamentalsRequest
): Promise<IStockFundamentals | IStockFundamentals[]> {
const cacheKey = this.getCacheKey(request);
const cached = this.getFromCache(cacheKey);
if (cached) {
console.log(`Cache hit for ${this.getRequestDescription(request)}`);
return cached;
}
const providers = this.getEnabledProviders();
if (providers.length === 0) {
throw new Error('No fundamentals providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData(request),
entry.config
);
entry.successCount++;
// Use provider-specific cache TTL or default
const ttl = entry.config.cacheTTL || this.cacheConfig.ttl;
this.addToCache(cacheKey, result, ttl);
console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`);
return result;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(
`Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}`
);
}
}
throw new Error(
`Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}`
);
}
/**
* Enrich fundamentals with calculated metrics using current price
*/
public async enrichWithPrice(
fundamentals: IStockFundamentals,
price: number
): Promise<IStockFundamentals> {
const enriched = { ...fundamentals };
// Calculate market cap: price × shares outstanding
if (fundamentals.sharesOutstanding) {
enriched.marketCap = price * fundamentals.sharesOutstanding;
}
// Calculate P/E ratio: price / EPS
if (fundamentals.earningsPerShareDiluted && fundamentals.earningsPerShareDiluted > 0) {
enriched.priceToEarnings = price / fundamentals.earningsPerShareDiluted;
}
// Calculate price-to-book: market cap / stockholders equity
if (enriched.marketCap && fundamentals.stockholdersEquity && fundamentals.stockholdersEquity > 0) {
enriched.priceToBook = enriched.marketCap / fundamentals.stockholdersEquity;
}
return enriched;
}
/**
* Enrich batch fundamentals with prices
*/
public async enrichBatchWithPrices(
fundamentalsList: IStockFundamentals[],
priceMap: Map<string, number>
): Promise<IStockFundamentals[]> {
return Promise.all(
fundamentalsList.map(fundamentals => {
const price = priceMap.get(fundamentals.ticker);
if (price) {
return this.enrichWithPrice(fundamentals, price);
}
return Promise.resolve(fundamentals);
})
);
}
/**
* Check health of all providers
*/
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
const health = new Map<string, boolean>();
for (const [name, entry] of this.providers) {
if (!entry.config.enabled) {
health.set(name, false);
continue;
}
try {
const isAvailable = await entry.provider.isAvailable();
health.set(name, isAvailable);
} catch (error) {
health.set(name, false);
console.error(`Health check failed for ${name}:`, error);
}
}
return health;
}
/**
* Get provider statistics
*/
public getProviderStats(): Map<
string,
{
successCount: number;
errorCount: number;
lastError?: string;
lastErrorTime?: Date;
}
> {
const stats = new Map();
for (const [name, entry] of this.providers) {
stats.set(name, {
successCount: entry.successCount,
errorCount: entry.errorCount,
lastError: entry.lastError?.message,
lastErrorTime: entry.lastErrorTime
});
}
return stats;
}
/**
* Clear all cached data
*/
public clearCache(): void {
this.cache.clear();
console.log('Fundamentals cache cleared');
}
/**
* Set cache TTL
*/
public setCacheTTL(ttl: number): void {
this.cacheConfig.ttl = ttl;
console.log(`Fundamentals cache TTL set to ${ttl}ms`);
}
/**
* Get cache statistics
*/
public getCacheStats(): {
size: number;
maxEntries: number;
ttl: number;
} {
return {
size: this.cache.size,
maxEntries: this.cacheConfig.maxEntries,
ttl: this.cacheConfig.ttl
};
}
/**
* Fetch with retry logic
*/
private async fetchWithRetry<T>(
fetchFn: () => Promise<T>,
config: IFundamentalsProviderConfig
): Promise<T> {
const maxAttempts = config.retryAttempts || 1;
let lastError: Error | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await fetchFn();
} catch (error) {
lastError = error as Error;
if (attempt < maxAttempts) {
const delay = (config.retryDelay || 1000) * attempt;
console.log(`Retry attempt ${attempt} after ${delay}ms`);
await plugins.smartdelay.delayFor(delay);
}
}
}
throw lastError || new Error('Unknown error during fetch');
}
/**
* Generate cache key for request
*/
private getCacheKey(request: IFundamentalsRequest): string {
switch (request.type) {
case 'fundamentals-current':
return `fundamentals:${request.ticker}`;
case 'fundamentals-batch':
const tickers = request.tickers.sort().join(',');
return `fundamentals-batch:${tickers}`;
default:
return `unknown:${JSON.stringify(request)}`;
}
}
/**
* Get from cache if not expired
*/
private getFromCache(key: string): IStockFundamentals | IStockFundamentals[] | null {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
// Check if cache entry has expired
const age = Date.now() - entry.timestamp.getTime();
if (entry.ttl !== Infinity && age > entry.ttl) {
this.cache.delete(key);
return null;
}
return entry.fundamentals;
}
/**
* Add to cache with TTL
*/
private addToCache(
key: string,
fundamentals: IStockFundamentals | IStockFundamentals[],
ttl?: number
): void {
// Enforce max entries limit
if (this.cache.size >= this.cacheConfig.maxEntries) {
// Remove oldest entry
const oldestKey = this.cache.keys().next().value;
if (oldestKey) {
this.cache.delete(oldestKey);
}
}
this.cache.set(key, {
fundamentals,
timestamp: new Date(),
ttl: ttl || this.cacheConfig.ttl
});
}
/**
* Get human-readable request description
*/
private getRequestDescription(request: IFundamentalsRequest): string {
switch (request.type) {
case 'fundamentals-current':
return `fundamentals for ${request.ticker}`;
case 'fundamentals-batch':
return `fundamentals for ${request.tickers.length} tickers`;
default:
return 'fundamentals data';
}
}
}

View File

@@ -0,0 +1,647 @@
import * as plugins from '../plugins.js';
import type { IStockProvider, IProviderConfig } from './interfaces/provider.js';
import type { IFundamentalsProvider, IFundamentalsProviderConfig, IStockFundamentals } from './interfaces/fundamentals.js';
import type { IStockPrice, IStockDataRequest as IPriceRequest } from './interfaces/stockprice.js';
import type { IStockData, IStockDataServiceConfig, ICompleteStockDataRequest, ICompleteStockDataBatchRequest } from './interfaces/stockdata.js';
interface IProviderEntry<T> {
provider: T;
config: IProviderConfig | IFundamentalsProviderConfig;
lastError?: Error;
lastErrorTime?: Date;
successCount: number;
errorCount: number;
}
interface ICacheEntry<T> {
data: T;
timestamp: Date;
ttl: number;
}
/**
* Unified service for managing both stock prices and fundamentals
* Provides automatic enrichment and convenient combined data access
*/
export class StockDataService {
private priceProviders = new Map<string, IProviderEntry<IStockProvider>>();
private fundamentalsProviders = new Map<string, IProviderEntry<IFundamentalsProvider>>();
private priceCache = new Map<string, ICacheEntry<IStockPrice | IStockPrice[]>>();
private fundamentalsCache = new Map<string, ICacheEntry<IStockFundamentals | IStockFundamentals[]>>();
private logger = console;
private config: Required<IStockDataServiceConfig> = {
cache: {
priceTTL: 24 * 60 * 60 * 1000, // 24 hours
fundamentalsTTL: 90 * 24 * 60 * 60 * 1000, // 90 days
maxEntries: 10000
},
timeout: {
price: 10000, // 10 seconds
fundamentals: 30000 // 30 seconds
}
};
constructor(config?: IStockDataServiceConfig) {
if (config) {
this.config = {
cache: { ...this.config.cache, ...config.cache },
timeout: { ...this.config.timeout, ...config.timeout }
};
}
}
// ========== Provider Management ==========
/**
* Register a price provider
*/
public registerPriceProvider(provider: IStockProvider, config?: IProviderConfig): void {
const defaultConfig: IProviderConfig = {
enabled: true,
priority: provider.priority,
timeout: this.config.timeout.price,
retryAttempts: 2,
retryDelay: 1000
};
const mergedConfig = { ...defaultConfig, ...config };
this.priceProviders.set(provider.name, {
provider,
config: mergedConfig,
successCount: 0,
errorCount: 0
});
console.log(`Registered price provider: ${provider.name}`);
}
/**
* Register a fundamentals provider
*/
public registerFundamentalsProvider(
provider: IFundamentalsProvider,
config?: IFundamentalsProviderConfig
): void {
const defaultConfig: IFundamentalsProviderConfig = {
enabled: true,
priority: provider.priority,
timeout: this.config.timeout.fundamentals,
retryAttempts: 2,
retryDelay: 1000,
cacheTTL: this.config.cache.fundamentalsTTL
};
const mergedConfig = { ...defaultConfig, ...config };
this.fundamentalsProviders.set(provider.name, {
provider,
config: mergedConfig,
successCount: 0,
errorCount: 0
});
console.log(`Registered fundamentals provider: ${provider.name}`);
}
/**
* Unregister a price provider
*/
public unregisterPriceProvider(providerName: string): void {
this.priceProviders.delete(providerName);
console.log(`Unregistered price provider: ${providerName}`);
}
/**
* Unregister a fundamentals provider
*/
public unregisterFundamentalsProvider(providerName: string): void {
this.fundamentalsProviders.delete(providerName);
console.log(`Unregistered fundamentals provider: ${providerName}`);
}
/**
* Get all registered price providers
*/
public getPriceProviders(): IStockProvider[] {
return Array.from(this.priceProviders.values()).map(entry => entry.provider);
}
/**
* Get all registered fundamentals providers
*/
public getFundamentalsProviders(): IFundamentalsProvider[] {
return Array.from(this.fundamentalsProviders.values()).map(entry => entry.provider);
}
/**
* Get enabled price providers sorted by priority
*/
private getEnabledPriceProviders(): IStockProvider[] {
return Array.from(this.priceProviders.values())
.filter(entry => entry.config.enabled)
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
.map(entry => entry.provider);
}
/**
* Get enabled fundamentals providers sorted by priority
*/
private getEnabledFundamentalsProviders(): IFundamentalsProvider[] {
return Array.from(this.fundamentalsProviders.values())
.filter(entry => entry.config.enabled)
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
.map(entry => entry.provider);
}
// ========== Data Fetching Methods ==========
/**
* Get current price for a single ticker
*/
public async getPrice(ticker: string): Promise<IStockPrice> {
const cacheKey = `price:${ticker}`;
const cached = this.getFromCache(this.priceCache, cacheKey);
if (cached) {
console.log(`Cache hit for price: ${ticker}`);
return cached as IStockPrice;
}
const providers = this.getEnabledPriceProviders();
if (providers.length === 0) {
throw new Error('No price providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.priceProviders.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData({ type: 'current', ticker }),
entry.config
);
entry.successCount++;
const price = result as IStockPrice;
this.addToCache(this.priceCache, cacheKey, price, this.config.cache.priceTTL);
console.log(`Successfully fetched price for ${ticker} from ${provider.name}`);
return price;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(`Provider ${provider.name} failed for ${ticker}: ${error.message}`);
}
}
throw new Error(
`Failed to fetch price for ${ticker} from all providers. Last error: ${lastError?.message}`
);
}
/**
* Get current prices for multiple tickers
*/
public async getPrices(tickers: string[]): Promise<IStockPrice[]> {
const cacheKey = `prices:${tickers.sort().join(',')}`;
const cached = this.getFromCache(this.priceCache, cacheKey);
if (cached) {
console.log(`Cache hit for prices: ${tickers.length} tickers`);
return cached as IStockPrice[];
}
const providers = this.getEnabledPriceProviders();
if (providers.length === 0) {
throw new Error('No price providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.priceProviders.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData({ type: 'batch', tickers }),
entry.config
);
entry.successCount++;
const prices = result as IStockPrice[];
this.addToCache(this.priceCache, cacheKey, prices, this.config.cache.priceTTL);
console.log(`Successfully fetched ${prices.length} prices from ${provider.name}`);
return prices;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(`Provider ${provider.name} failed for batch prices: ${error.message}`);
}
}
throw new Error(
`Failed to fetch prices for ${tickers.length} tickers from all providers. Last error: ${lastError?.message}`
);
}
/**
* Get fundamentals for a single ticker
*/
public async getFundamentals(ticker: string): Promise<IStockFundamentals> {
const cacheKey = `fundamentals:${ticker}`;
const cached = this.getFromCache(this.fundamentalsCache, cacheKey);
if (cached) {
console.log(`Cache hit for fundamentals: ${ticker}`);
return cached as IStockFundamentals;
}
const providers = this.getEnabledFundamentalsProviders();
if (providers.length === 0) {
throw new Error('No fundamentals providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.fundamentalsProviders.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData({ type: 'fundamentals-current', ticker }),
entry.config
);
entry.successCount++;
const fundamentals = result as IStockFundamentals;
const ttl = (entry.config as IFundamentalsProviderConfig).cacheTTL || this.config.cache.fundamentalsTTL;
this.addToCache(this.fundamentalsCache, cacheKey, fundamentals, ttl);
console.log(`Successfully fetched fundamentals for ${ticker} from ${provider.name}`);
return fundamentals;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(`Provider ${provider.name} failed for ${ticker} fundamentals: ${error.message}`);
}
}
throw new Error(
`Failed to fetch fundamentals for ${ticker} from all providers. Last error: ${lastError?.message}`
);
}
/**
* Get fundamentals for multiple tickers
*/
public async getBatchFundamentals(tickers: string[]): Promise<IStockFundamentals[]> {
const cacheKey = `fundamentals-batch:${tickers.sort().join(',')}`;
const cached = this.getFromCache(this.fundamentalsCache, cacheKey);
if (cached) {
console.log(`Cache hit for batch fundamentals: ${tickers.length} tickers`);
return cached as IStockFundamentals[];
}
const providers = this.getEnabledFundamentalsProviders();
if (providers.length === 0) {
throw new Error('No fundamentals providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.fundamentalsProviders.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData({ type: 'fundamentals-batch', tickers }),
entry.config
);
entry.successCount++;
const fundamentals = result as IStockFundamentals[];
const ttl = (entry.config as IFundamentalsProviderConfig).cacheTTL || this.config.cache.fundamentalsTTL;
this.addToCache(this.fundamentalsCache, cacheKey, fundamentals, ttl);
console.log(`Successfully fetched ${fundamentals.length} fundamentals from ${provider.name}`);
return fundamentals;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(`Provider ${provider.name} failed for batch fundamentals: ${error.message}`);
}
}
throw new Error(
`Failed to fetch fundamentals for ${tickers.length} tickers from all providers. Last error: ${lastError?.message}`
);
}
/**
* ✨ Get complete stock data (price + fundamentals) with automatic enrichment
*/
public async getStockData(request: string | ICompleteStockDataRequest): Promise<IStockData> {
const normalizedRequest = typeof request === 'string'
? { ticker: request, includeFundamentals: true, enrichFundamentals: true }
: { includeFundamentals: true, enrichFundamentals: true, ...request };
const price = await this.getPrice(normalizedRequest.ticker);
let fundamentals: IStockFundamentals | undefined;
if (normalizedRequest.includeFundamentals) {
try {
fundamentals = await this.getFundamentals(normalizedRequest.ticker);
// Enrich fundamentals with price calculations
if (normalizedRequest.enrichFundamentals && fundamentals) {
fundamentals = this.enrichWithPrice(fundamentals, price.price);
}
} catch (error) {
console.warn(`Failed to fetch fundamentals for ${normalizedRequest.ticker}: ${error.message}`);
// Continue without fundamentals
}
}
return {
ticker: normalizedRequest.ticker,
price,
fundamentals,
fetchedAt: new Date()
};
}
/**
* ✨ Get complete stock data for multiple tickers with automatic enrichment
*/
public async getBatchStockData(request: string[] | ICompleteStockDataBatchRequest): Promise<IStockData[]> {
const normalizedRequest = Array.isArray(request)
? { tickers: request, includeFundamentals: true, enrichFundamentals: true }
: { includeFundamentals: true, enrichFundamentals: true, ...request };
const prices = await this.getPrices(normalizedRequest.tickers);
const priceMap = new Map(prices.map(p => [p.ticker, p]));
let fundamentalsMap = new Map<string, IStockFundamentals>();
if (normalizedRequest.includeFundamentals) {
try {
const fundamentals = await this.getBatchFundamentals(normalizedRequest.tickers);
// Enrich with prices if requested
if (normalizedRequest.enrichFundamentals) {
for (const fund of fundamentals) {
const price = priceMap.get(fund.ticker);
if (price) {
fundamentalsMap.set(fund.ticker, this.enrichWithPrice(fund, price.price));
} else {
fundamentalsMap.set(fund.ticker, fund);
}
}
} else {
fundamentalsMap = new Map(fundamentals.map(f => [f.ticker, f]));
}
} catch (error) {
console.warn(`Failed to fetch batch fundamentals: ${error.message}`);
// Continue without fundamentals
}
}
return normalizedRequest.tickers.map(ticker => ({
ticker,
price: priceMap.get(ticker)!,
fundamentals: fundamentalsMap.get(ticker),
fetchedAt: new Date()
}));
}
// ========== Helper Methods ==========
/**
* Enrich fundamentals with calculated metrics using current price
*/
private enrichWithPrice(fundamentals: IStockFundamentals, price: number): IStockFundamentals {
const enriched = { ...fundamentals };
// Calculate market cap: price × shares outstanding
if (fundamentals.sharesOutstanding) {
enriched.marketCap = price * fundamentals.sharesOutstanding;
}
// Calculate P/E ratio: price / EPS
if (fundamentals.earningsPerShareDiluted && fundamentals.earningsPerShareDiluted > 0) {
enriched.priceToEarnings = price / fundamentals.earningsPerShareDiluted;
}
// Calculate price-to-book: market cap / stockholders equity
if (enriched.marketCap && fundamentals.stockholdersEquity && fundamentals.stockholdersEquity > 0) {
enriched.priceToBook = enriched.marketCap / fundamentals.stockholdersEquity;
}
return enriched;
}
/**
* Fetch with retry logic
*/
private async fetchWithRetry<T>(
fetchFn: () => Promise<T>,
config: IProviderConfig | IFundamentalsProviderConfig
): Promise<T> {
const maxAttempts = config.retryAttempts || 1;
let lastError: Error | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await fetchFn();
} catch (error) {
lastError = error as Error;
if (attempt < maxAttempts) {
const delay = (config.retryDelay || 1000) * attempt;
console.log(`Retry attempt ${attempt} after ${delay}ms`);
await plugins.smartdelay.delayFor(delay);
}
}
}
throw lastError || new Error('Unknown error during fetch');
}
/**
* Get from cache if not expired
*/
private getFromCache<T>(cache: Map<string, ICacheEntry<T>>, key: string): T | null {
const entry = cache.get(key);
if (!entry) {
return null;
}
// Check if cache entry has expired
const age = Date.now() - entry.timestamp.getTime();
if (entry.ttl !== Infinity && age > entry.ttl) {
cache.delete(key);
return null;
}
return entry.data;
}
/**
* Add to cache with TTL
*/
private addToCache<T>(cache: Map<string, ICacheEntry<T>>, key: string, data: T, ttl: number): void {
// Enforce max entries limit
if (cache.size >= this.config.cache.maxEntries) {
// Remove oldest entry
const oldestKey = cache.keys().next().value;
if (oldestKey) {
cache.delete(oldestKey);
}
}
cache.set(key, {
data,
timestamp: new Date(),
ttl
});
}
// ========== Health & Statistics ==========
/**
* Check health of all providers (both price and fundamentals)
*/
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
const health = new Map<string, boolean>();
// Check price providers
for (const [name, entry] of this.priceProviders) {
if (!entry.config.enabled) {
health.set(`${name} (price)`, false);
continue;
}
try {
const isAvailable = await entry.provider.isAvailable();
health.set(`${name} (price)`, isAvailable);
} catch (error) {
health.set(`${name} (price)`, false);
console.error(`Health check failed for ${name}:`, error);
}
}
// Check fundamentals providers
for (const [name, entry] of this.fundamentalsProviders) {
if (!entry.config.enabled) {
health.set(`${name} (fundamentals)`, false);
continue;
}
try {
const isAvailable = await entry.provider.isAvailable();
health.set(`${name} (fundamentals)`, isAvailable);
} catch (error) {
health.set(`${name} (fundamentals)`, false);
console.error(`Health check failed for ${name}:`, error);
}
}
return health;
}
/**
* Get statistics for all providers
*/
public getProviderStats(): Map<
string,
{
type: 'price' | 'fundamentals';
successCount: number;
errorCount: number;
lastError?: string;
lastErrorTime?: Date;
}
> {
const stats = new Map();
// Price provider stats
for (const [name, entry] of this.priceProviders) {
stats.set(name, {
type: 'price',
successCount: entry.successCount,
errorCount: entry.errorCount,
lastError: entry.lastError?.message,
lastErrorTime: entry.lastErrorTime
});
}
// Fundamentals provider stats
for (const [name, entry] of this.fundamentalsProviders) {
stats.set(name, {
type: 'fundamentals',
successCount: entry.successCount,
errorCount: entry.errorCount,
lastError: entry.lastError?.message,
lastErrorTime: entry.lastErrorTime
});
}
return stats;
}
/**
* Clear all caches
*/
public clearCache(): void {
this.priceCache.clear();
this.fundamentalsCache.clear();
console.log('All caches cleared');
}
/**
* Get cache statistics
*/
public getCacheStats(): {
priceCache: { size: number; ttl: number };
fundamentalsCache: { size: number; ttl: number };
maxEntries: number;
} {
return {
priceCache: {
size: this.priceCache.size,
ttl: this.config.cache.priceTTL
},
fundamentalsCache: {
size: this.fundamentalsCache.size,
ttl: this.config.cache.fundamentalsTTL
},
maxEntries: this.config.cache.maxEntries
};
}
}

View File

@@ -0,0 +1,525 @@
import * as plugins from '../plugins.js';
import type { IStockProvider, IProviderConfig, IProviderRegistry } from './interfaces/provider.js';
import type {
IStockPrice,
IStockPriceError,
IStockDataRequest,
IStockCurrentRequest,
IStockHistoricalRequest,
IStockIntradayRequest,
IStockBatchCurrentRequest,
TIntervalType
} from './interfaces/stockprice.js';
// Simple request interfaces for convenience methods
interface ISimpleQuoteRequest {
ticker: string;
}
interface ISimpleBatchRequest {
tickers: string[];
}
interface IProviderEntry {
provider: IStockProvider;
config: IProviderConfig;
lastError?: Error;
lastErrorTime?: Date;
successCount: number;
errorCount: number;
}
interface ICacheEntry {
price: IStockPrice | IStockPrice[];
timestamp: Date;
ttl: number; // Specific TTL for this entry
}
export class StockPriceService implements IProviderRegistry {
private providers = new Map<string, IProviderEntry>();
private cache = new Map<string, ICacheEntry>();
private logger = console;
private cacheConfig = {
ttl: 60000, // 60 seconds default (for backward compatibility)
maxEntries: 10000 // Increased for historical data
};
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
if (cacheConfig) {
this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
}
}
/**
* Get data-type aware TTL for smart caching
*/
private getCacheTTL(dataType: 'eod' | 'historical' | 'intraday' | 'live', interval?: TIntervalType): number {
switch (dataType) {
case 'historical':
return Infinity; // Historical data never changes
case 'eod':
return 24 * 60 * 60 * 1000; // 24 hours (EOD is static after market close)
case 'intraday':
// Match cache TTL to interval
return this.getIntervalMs(interval);
case 'live':
return 30 * 1000; // 30 seconds for live data
default:
return this.cacheConfig.ttl; // Fallback to default
}
}
/**
* Convert interval to milliseconds
*/
private getIntervalMs(interval?: TIntervalType): number {
if (!interval) return 60 * 1000; // Default 1 minute
const intervalMap: Record<TIntervalType, number> = {
'1min': 60 * 1000,
'5min': 5 * 60 * 1000,
'10min': 10 * 60 * 1000,
'15min': 15 * 60 * 1000,
'30min': 30 * 60 * 1000,
'1hour': 60 * 60 * 1000
};
return intervalMap[interval] || 60 * 1000;
}
public register(provider: IStockProvider, config?: IProviderConfig): void {
const defaultConfig: IProviderConfig = {
enabled: true,
priority: provider.priority,
timeout: 10000,
retryAttempts: 2,
retryDelay: 1000
};
const mergedConfig = { ...defaultConfig, ...config };
this.providers.set(provider.name, {
provider,
config: mergedConfig,
successCount: 0,
errorCount: 0
});
console.log(`Registered provider: ${provider.name}`);
}
public unregister(providerName: string): void {
this.providers.delete(providerName);
console.log(`Unregistered provider: ${providerName}`);
}
public getProvider(name: string): IStockProvider | undefined {
return this.providers.get(name)?.provider;
}
public getAllProviders(): IStockProvider[] {
return Array.from(this.providers.values()).map(entry => entry.provider);
}
public getEnabledProviders(): IStockProvider[] {
return Array.from(this.providers.values())
.filter(entry => entry.config.enabled)
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
.map(entry => entry.provider);
}
/**
* Convenience method: Get current price for a single ticker
*/
public async getPrice(request: ISimpleQuoteRequest): Promise<IStockPrice> {
const result = await this.getData({
type: 'current',
ticker: request.ticker
});
return result as IStockPrice;
}
/**
* Convenience method: Get current prices for multiple tickers
*/
public async getPrices(request: ISimpleBatchRequest): Promise<IStockPrice[]> {
const result = await this.getData({
type: 'batch',
tickers: request.tickers
});
return result as IStockPrice[];
}
/**
* New unified data fetching method supporting all request types
*/
public async getData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
const cacheKey = this.getDataCacheKey(request);
// For intraday requests without date filter, ALWAYS try incremental fetch
// This ensures we check for new data even if cache hasn't expired
if (request.type === 'intraday' && !request.date) {
const incrementalResult = await this.tryIncrementalFetch(request, cacheKey);
if (incrementalResult) {
return incrementalResult;
}
// If incremental fetch returns null, continue to normal fetch below
} else {
// For other request types (historical, current, batch), use simple cache
const cached = this.getFromCache(cacheKey);
if (cached) {
console.log(`Cache hit for ${this.getRequestDescription(request)}`);
return cached;
}
}
const providers = this.getEnabledProviders();
if (providers.length === 0) {
throw new Error('No stock price providers available');
}
let lastError: Error | undefined;
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
try {
const result = await this.fetchWithRetry(
() => provider.fetchData(request),
entry.config
) as IStockPrice | IStockPrice[];
entry.successCount++;
// Determine TTL based on request type
const ttl = this.getRequestTTL(request, result);
this.addToCache(cacheKey, result, ttl);
console.log(`Successfully fetched ${this.getRequestDescription(request)} from ${provider.name}`);
return result;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
lastError = error as Error;
console.warn(
`Provider ${provider.name} failed for ${this.getRequestDescription(request)}: ${error.message}`
);
}
}
throw new Error(
`Failed to fetch ${this.getRequestDescription(request)} from all providers. Last error: ${lastError?.message}`
);
}
/**
* Try incremental fetch: Only fetch NEW data since last cached timestamp
* Returns merged result if successful, null if incremental fetch not applicable
*/
private async tryIncrementalFetch(
request: IStockDataRequest,
cacheKey: string
): Promise<IStockPrice[] | null> {
// Only applicable for intraday requests without date filter
if (request.type !== 'intraday' || request.date) {
return null;
}
// Check if we have similar cached data (same ticker, interval, but any limit/date)
const baseKey = `intraday:${request.ticker}:${request.interval}:latest`;
let cachedData: IStockPrice[] | null = null;
let matchedKey: string | null = null;
// Find any cached intraday data for this ticker+interval
for (const [key, entry] of this.cache.entries()) {
if (key.startsWith(baseKey)) {
const age = Date.now() - entry.timestamp.getTime();
if (entry.ttl !== Infinity && age > entry.ttl) {
continue; // Expired
}
cachedData = Array.isArray(entry.price) ? entry.price as IStockPrice[] : null;
matchedKey = key;
break;
}
}
if (!cachedData || cachedData.length === 0) {
return null; // No cached data to build on
}
// Find latest timestamp in cached data
const latestCached = cachedData.reduce((latest, price) => {
return price.timestamp > latest ? price.timestamp : latest;
}, new Date(0));
// Freshness check: If latest data is less than 1 minute old, just return cache
const dataAge = Date.now() - latestCached.getTime();
const freshnessThreshold = 60 * 1000; // 1 minute
if (dataAge < freshnessThreshold) {
console.log(`🔄 Incremental cache: Latest data is ${Math.round(dataAge / 1000)}s old (< 1min), returning cached data`);
return cachedData;
}
console.log(`🔄 Incremental cache: Found ${cachedData.length} cached records, latest: ${latestCached.toISOString()} (${Math.round(dataAge / 1000)}s old)`);
// Fetch only NEW data since latest cached timestamp
// Create a modified request with date filter
const modifiedRequest: IStockIntradayRequest = {
...request,
date: latestCached // Fetch from this date forward
};
const providers = this.getEnabledProviders();
for (const provider of providers) {
const entry = this.providers.get(provider.name)!;
try {
const newData = await this.fetchWithRetry(
() => provider.fetchData(modifiedRequest),
entry.config
) as IStockPrice[];
entry.successCount++;
// Filter out data at or before latest cached timestamp (avoid duplicates)
const filteredNew = newData.filter(p => p.timestamp > latestCached);
if (filteredNew.length === 0) {
console.log(`🔄 Incremental cache: No new data since ${latestCached.toISOString()}, using cache`);
return cachedData;
}
console.log(`🔄 Incremental cache: Fetched ${filteredNew.length} new records since ${latestCached.toISOString()}`);
// Merge cached + new data
const merged = [...cachedData, ...filteredNew];
// Sort by timestamp (ascending)
merged.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
// Deduplicate by timestamp (keep latest)
const deduped = this.deduplicateByTimestamp(merged);
// Apply limit if specified in original request
const effectiveLimit = request.limit || deduped.length;
const result = deduped.slice(-effectiveLimit); // Take most recent N
// Update cache with merged result
const ttl = this.getRequestTTL(request, result);
this.addToCache(cacheKey, result, ttl);
console.log(`🔄 Incremental cache: Returning ${result.length} total records (${cachedData.length} cached + ${filteredNew.length} new)`);
return result;
} catch (error) {
entry.errorCount++;
entry.lastError = error as Error;
entry.lastErrorTime = new Date();
console.warn(`Incremental fetch failed for ${provider.name}, falling back to full fetch`);
continue; // Try next provider or fall back to normal fetch
}
}
return null; // Incremental fetch failed, fall back to normal fetch
}
/**
* Deduplicate array of prices by timestamp, keeping the latest data for each timestamp
*/
private deduplicateByTimestamp(prices: IStockPrice[]): IStockPrice[] {
const seen = new Map<number, IStockPrice>();
for (const price of prices) {
const ts = price.timestamp.getTime();
const existing = seen.get(ts);
// Keep the entry with the latest fetchedAt (most recent data)
if (!existing || price.fetchedAt > existing.fetchedAt) {
seen.set(ts, price);
}
}
return Array.from(seen.values());
}
/**
* Get TTL based on request type and result
*/
private getRequestTTL(request: IStockDataRequest, result: IStockPrice | IStockPrice[]): number {
switch (request.type) {
case 'historical':
return Infinity; // Historical data never changes
case 'current':
return this.getCacheTTL('eod');
case 'batch':
return this.getCacheTTL('eod');
case 'intraday':
return this.getCacheTTL('intraday', request.interval);
default:
return this.cacheConfig.ttl;
}
}
/**
* Get human-readable description of request
*/
private getRequestDescription(request: IStockDataRequest): string {
switch (request.type) {
case 'current':
return `current price for ${request.ticker}${request.exchange ? ` on ${request.exchange}` : ''}`;
case 'historical':
return `historical prices for ${request.ticker} from ${request.from.toISOString().split('T')[0]} to ${request.to.toISOString().split('T')[0]}`;
case 'intraday':
return `intraday ${request.interval} prices for ${request.ticker}`;
case 'batch':
return `batch prices for ${request.tickers.length} tickers`;
default:
return 'data';
}
}
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
const health = new Map<string, boolean>();
for (const [name, entry] of this.providers) {
if (!entry.config.enabled) {
health.set(name, false);
continue;
}
try {
const isAvailable = await entry.provider.isAvailable();
health.set(name, isAvailable);
} catch (error) {
health.set(name, false);
console.error(`Health check failed for ${name}:`, error);
}
}
return health;
}
public getProviderStats(): Map<string, {
successCount: number;
errorCount: number;
lastError?: string;
lastErrorTime?: Date;
}> {
const stats = new Map();
for (const [name, entry] of this.providers) {
stats.set(name, {
successCount: entry.successCount,
errorCount: entry.errorCount,
lastError: entry.lastError?.message,
lastErrorTime: entry.lastErrorTime
});
}
return stats;
}
public clearCache(): void {
this.cache.clear();
console.log('Cache cleared');
}
public setCacheTTL(ttl: number): void {
this.cacheConfig.ttl = ttl;
console.log(`Cache TTL set to ${ttl}ms`);
}
private async fetchWithRetry<T>(
fetchFn: () => Promise<T>,
config: IProviderConfig
): Promise<T> {
const maxAttempts = config.retryAttempts || 1;
let lastError: Error | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await fetchFn();
} catch (error) {
lastError = error as Error;
if (attempt < maxAttempts) {
const delay = (config.retryDelay || 1000) * attempt;
console.log(`Retry attempt ${attempt} after ${delay}ms`);
await plugins.smartdelay.delayFor(delay);
}
}
}
throw lastError || new Error('Unknown error during fetch');
}
/**
* New cache key generation for discriminated union requests
*/
private getDataCacheKey(request: IStockDataRequest): string {
switch (request.type) {
case 'current':
return `current:${request.ticker}${request.exchange ? `:${request.exchange}` : ''}`;
case 'historical':
const fromStr = request.from.toISOString().split('T')[0];
const toStr = request.to.toISOString().split('T')[0];
return `historical:${request.ticker}:${fromStr}:${toStr}${request.exchange ? `:${request.exchange}` : ''}`;
case 'intraday':
const dateStr = request.date ? request.date.toISOString().split('T')[0] : 'latest';
const limitStr = request.limit ? `:limit${request.limit}` : '';
return `intraday:${request.ticker}:${request.interval}:${dateStr}${limitStr}${request.exchange ? `:${request.exchange}` : ''}`;
case 'batch':
const tickers = request.tickers.sort().join(',');
return `batch:${tickers}${request.exchange ? `:${request.exchange}` : ''}`;
default:
return `unknown:${JSON.stringify(request)}`;
}
}
private getFromCache(key: string): IStockPrice | IStockPrice[] | null {
const entry = this.cache.get(key);
if (!entry) {
return null;
}
// Check if cache entry has expired
const age = Date.now() - entry.timestamp.getTime();
if (entry.ttl !== Infinity && age > entry.ttl) {
this.cache.delete(key);
return null;
}
return entry.price;
}
private addToCache(key: string, price: IStockPrice | IStockPrice[], ttl?: number): void {
// Deduplicate array entries by timestamp before caching
if (Array.isArray(price)) {
const beforeCount = price.length;
price = this.deduplicateByTimestamp(price);
if (price.length < beforeCount) {
console.log(`Deduplicated ${beforeCount - price.length} duplicate timestamps in cache entry for ${key}`);
}
}
// Enforce max entries limit
if (this.cache.size >= this.cacheConfig.maxEntries) {
// Remove oldest entry
const oldestKey = this.cache.keys().next().value;
if (oldestKey) {
this.cache.delete(oldestKey);
}
}
this.cache.set(key, {
price,
timestamp: new Date(),
ttl: ttl || this.cacheConfig.ttl
});
}
}

19
ts/stocks/index.ts Normal file
View File

@@ -0,0 +1,19 @@
// Export all interfaces
export * from './interfaces/stockprice.js';
export * from './interfaces/provider.js';
export * from './interfaces/fundamentals.js';
export * from './interfaces/stockdata.js';
// Export main services
export * from './classes.stockservice.js';
export * from './classes.fundamentalsservice.js';
export * from './classes.stockdataservice.js'; // ✨ New unified service
// Export base service (for advanced use cases)
export * from './classes.baseproviderservice.js';
// Export providers
export * from './providers/provider.yahoo.js';
export * from './providers/provider.marketstack.js';
export * from './providers/provider.secedgar.js';
export * from './providers/provider.coingecko.js';

View File

@@ -0,0 +1,102 @@
/**
* Interfaces for stock fundamental data (financials from SEC filings)
* Separate from stock price data (OHLCV) to maintain clean architecture
*/
// Request types for fundamental data
export interface IFundamentalsCurrentRequest {
type: 'fundamentals-current';
ticker: string;
}
export interface IFundamentalsBatchRequest {
type: 'fundamentals-batch';
tickers: string[];
}
export type IFundamentalsRequest =
| IFundamentalsCurrentRequest
| IFundamentalsBatchRequest;
/**
* Stock fundamental data from SEC filings (10-K, 10-Q)
* Contains financial metrics like EPS, Revenue, Assets, etc.
*/
export interface IStockFundamentals {
ticker: string;
cik: string;
companyName: string;
provider: string;
timestamp: Date;
fetchedAt: Date;
// Per-share metrics
earningsPerShareBasic?: number;
earningsPerShareDiluted?: number;
sharesOutstanding?: number;
weightedAverageSharesOutstanding?: number;
// Income statement (annual USD)
revenue?: number;
netIncome?: number;
operatingIncome?: number;
grossProfit?: number;
costOfRevenue?: number;
// Balance sheet (annual USD)
assets?: number;
liabilities?: number;
stockholdersEquity?: number;
cash?: number;
propertyPlantEquipment?: number;
// Calculated metrics (requires current price)
marketCap?: number; // price × sharesOutstanding
priceToEarnings?: number; // price / EPS
priceToBook?: number; // marketCap / stockholdersEquity
// Metadata
fiscalYear?: string;
fiscalQuarter?: string;
filingDate?: Date;
form?: '10-K' | '10-Q' | string;
}
/**
* Provider interface for fetching fundamental data
* Parallel to IStockProvider but for fundamentals instead of prices
*/
export interface IFundamentalsProvider {
name: string;
priority: number;
fetchData(request: IFundamentalsRequest): Promise<IStockFundamentals | IStockFundamentals[]>;
isAvailable(): Promise<boolean>;
readonly requiresAuth: boolean;
readonly rateLimit?: {
requestsPerMinute: number;
requestsPerDay?: number;
};
}
/**
* Configuration for fundamentals providers
*/
export interface IFundamentalsProviderConfig {
enabled: boolean;
priority?: number;
timeout?: number;
retryAttempts?: number;
retryDelay?: number;
cacheTTL?: number; // Custom cache TTL for this provider
}
/**
* Registry for managing fundamental data providers
*/
export interface IFundamentalsProviderRegistry {
register(provider: IFundamentalsProvider, config?: IFundamentalsProviderConfig): void;
unregister(providerName: string): void;
getProvider(name: string): IFundamentalsProvider | undefined;
getAllProviders(): IFundamentalsProvider[];
getEnabledProviders(): IFundamentalsProvider[];
}

View File

@@ -0,0 +1,37 @@
import type { IStockPrice, IStockDataRequest } from './stockprice.js';
export interface IStockProvider {
name: string;
priority: number;
fetchData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]>;
isAvailable(): Promise<boolean>;
supportsMarket?(market: string): boolean;
supportsTicker?(ticker: string): boolean;
readonly requiresAuth: boolean;
readonly rateLimit?: {
requestsPerMinute: number;
requestsPerDay?: number;
};
}
export interface IProviderConfig {
enabled: boolean;
priority?: number;
apiKey?: string;
timeout?: number;
retryAttempts?: number;
retryDelay?: number;
maxRecords?: number; // Maximum records to fetch per request (default: 10000)
defaultIntradayLimit?: number; // Default limit for intraday requests without explicit limit (default: 1000)
}
export interface IProviderRegistry {
register(provider: IStockProvider, config?: IProviderConfig): void;
unregister(providerName: string): void;
getProvider(name: string): IStockProvider | undefined;
getAllProviders(): IStockProvider[];
getEnabledProviders(): IStockProvider[];
}

View File

@@ -0,0 +1,65 @@
import type { IStockPrice } from './stockprice.js';
import type { IStockFundamentals } from './fundamentals.js';
/**
* Combined stock data with price and fundamentals
* All calculated metrics (market cap, P/E, P/B) are automatically included
*/
export interface IStockData {
/** Stock ticker symbol */
ticker: string;
/** Price information */
price: IStockPrice;
/** Fundamental data (optional - may not be available for all stocks) */
fundamentals?: IStockFundamentals;
/** When this combined data was fetched */
fetchedAt: Date;
}
/**
* Configuration for StockDataService
*/
export interface IStockDataServiceConfig {
/** Cache configuration */
cache?: {
/** TTL for price data (default: 24 hours) */
priceTTL?: number;
/** TTL for fundamentals data (default: 90 days) */
fundamentalsTTL?: number;
/** Max cache entries (default: 10000) */
maxEntries?: number;
};
/** Provider timeouts */
timeout?: {
/** Timeout for price providers (default: 10000ms) */
price?: number;
/** Timeout for fundamentals providers (default: 30000ms) */
fundamentals?: number;
};
}
/**
* Request type for getting complete stock data
*/
export interface ICompleteStockDataRequest {
ticker: string;
/** Whether to include fundamentals (default: true) */
includeFundamentals?: boolean;
/** Whether to enrich fundamentals with price calculations (default: true) */
enrichFundamentals?: boolean;
}
/**
* Batch request for multiple stocks
*/
export interface ICompleteStockDataBatchRequest {
tickers: string[];
/** Whether to include fundamentals (default: true) */
includeFundamentals?: boolean;
/** Whether to enrich fundamentals with price calculations (default: true) */
enrichFundamentals?: boolean;
}

View File

@@ -0,0 +1,94 @@
// Enhanced stock price interface with additional OHLCV data
export interface IStockPrice {
ticker: string;
price: number;
currency: string;
change: number;
changePercent: number;
previousClose: number;
timestamp: Date;
provider: string;
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
exchange?: string;
exchangeName?: string;
// Phase 1 enhancements
volume?: number; // Trading volume
open?: number; // Opening price
high?: number; // Day high
low?: number; // Day low
adjusted?: boolean; // If price is split/dividend adjusted
dataType: 'eod' | 'intraday' | 'live'; // What kind of data this is
fetchedAt: Date; // When we fetched (vs data timestamp)
// Company identification
companyName?: string; // Company name (e.g., "Apple Inc.")
companyFullName?: string; // Full company name with exchange (e.g., "Apple Inc. (NASDAQ:AAPL)")
}
export interface IStockPriceError {
ticker: string;
error: string;
provider: string;
timestamp: Date;
}
// Pagination support for large datasets
export interface IPaginatedResponse<T> {
data: T[];
pagination: {
currentPage: number;
totalPages: number;
totalRecords: number;
hasMore: boolean;
limit: number;
offset: number;
};
}
// Phase 1: Discriminated union types for different request types
export type TIntervalType = '1min' | '5min' | '10min' | '15min' | '30min' | '1hour';
export type TSortOrder = 'ASC' | 'DESC';
// Current price request (latest EOD or live)
export interface IStockCurrentRequest {
type: 'current';
ticker: string;
exchange?: string; // MIC code like 'XNAS', 'XNYS', 'XLON'
}
// Historical price request (date range)
export interface IStockHistoricalRequest {
type: 'historical';
ticker: string;
from: Date;
to: Date;
exchange?: string;
sort?: TSortOrder;
limit?: number; // Max results per page (default 1000)
offset?: number; // For pagination
}
// Intraday price request (real-time intervals)
export interface IStockIntradayRequest {
type: 'intraday';
ticker: string;
interval: TIntervalType;
exchange?: string;
limit?: number; // Number of bars to return
date?: Date; // Specific date for historical intraday
}
// Batch current prices request
export interface IStockBatchCurrentRequest {
type: 'batch';
tickers: string[];
exchange?: string;
}
// Union type for all stock data requests
export type IStockDataRequest =
| IStockCurrentRequest
| IStockHistoricalRequest
| IStockIntradayRequest
| IStockBatchCurrentRequest;

View File

@@ -0,0 +1,791 @@
import * as plugins from '../../plugins.js';
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
import type {
IStockPrice,
IStockDataRequest,
IStockCurrentRequest,
IStockHistoricalRequest,
IStockIntradayRequest,
IStockBatchCurrentRequest
} from '../interfaces/stockprice.js';
/**
* Custom error for rate limit exceeded responses
*/
class RateLimitError extends Error {
constructor(
message: string,
public waitTime: number,
public retryAfter?: number
) {
super(message);
this.name = 'RateLimitError';
}
}
/**
* Rate limiter for CoinGecko API
* Free tier (Demo): 30 requests per minute
* Without registration: 5-15 requests per minute
*/
class RateLimiter {
private requestTimes: number[] = [];
private maxRequestsPerMinute: number;
private consecutiveRateLimitErrors: number = 0;
constructor(maxRequestsPerMinute: number = 30) {
this.maxRequestsPerMinute = maxRequestsPerMinute;
}
public async waitForSlot(): Promise<void> {
const now = Date.now();
const oneMinuteAgo = now - 60000;
// Remove requests older than 1 minute
this.requestTimes = this.requestTimes.filter(time => time > oneMinuteAgo);
// If we've hit the limit, wait
if (this.requestTimes.length >= this.maxRequestsPerMinute) {
const oldestRequest = this.requestTimes[0];
const waitTime = 60000 - (now - oldestRequest) + 100; // +100ms buffer
await plugins.smartdelay.delayFor(waitTime);
return this.waitForSlot(); // Recursively check again
}
// Record this request
this.requestTimes.push(now);
}
/**
* Get time in milliseconds until next request slot is available
*/
public getTimeUntilNextSlot(): number {
const now = Date.now();
const oneMinuteAgo = now - 60000;
// Clean old requests
const recentRequests = this.requestTimes.filter(time => time > oneMinuteAgo);
if (recentRequests.length < this.maxRequestsPerMinute) {
return 0; // Slot available now
}
// Calculate wait time until oldest request expires
const oldestRequest = recentRequests[0];
return Math.max(0, 60000 - (now - oldestRequest) + 100);
}
/**
* Handle rate limit error with exponential backoff
* Returns wait time in milliseconds
*/
public handleRateLimitError(): number {
this.consecutiveRateLimitErrors++;
// Exponential backoff: 1s, 2s, 4s, 8s, 16s, 32s, 60s (max)
const baseWait = 1000; // 1 second
const exponent = this.consecutiveRateLimitErrors - 1;
const backoff = Math.min(
baseWait * Math.pow(2, exponent),
60000 // max 60 seconds
);
// After 3 consecutive 429s, reduce rate limit to 80% as safety measure
if (this.consecutiveRateLimitErrors >= 3) {
const newLimit = Math.floor(this.maxRequestsPerMinute * 0.8);
if (newLimit < this.maxRequestsPerMinute) {
console.warn(
`Adjusting rate limit from ${this.maxRequestsPerMinute} to ${newLimit} requests/min due to repeated 429 errors`
);
this.maxRequestsPerMinute = newLimit;
}
}
return backoff;
}
/**
* Reset consecutive error count on successful request
*/
public resetErrors(): void {
if (this.consecutiveRateLimitErrors > 0) {
this.consecutiveRateLimitErrors = 0;
}
}
}
/**
* Interface for coin list response
*/
interface ICoinListItem {
id: string;
symbol: string;
name: string;
}
/**
* CoinGecko Crypto Price Provider
*
* Documentation: https://docs.coingecko.com/v3.0.1/reference/endpoint-overview
*
* Features:
* - Current crypto prices (single and batch)
* - Historical price data with OHLCV
* - 13M+ tokens, 240+ networks, 1600+ exchanges
* - Accepts both ticker symbols (BTC, ETH) and CoinGecko IDs (bitcoin, ethereum)
* - 24/7 market data (crypto never closes)
*
* Rate Limits:
* - Free tier (no key): 5-15 requests/minute
* - Demo plan (free with registration): ~30 requests/minute, 10,000/month
* - Paid plans: Higher limits
*
* API Authentication:
* - Optional API key for Demo/paid plans
* - Header: x-cg-demo-api-key (Demo) or x-cg-pro-api-key (paid)
*/
export class CoinGeckoProvider implements IStockProvider {
public name = 'CoinGecko';
public priority = 90; // High priority for crypto, between Yahoo (100) and Marketstack (80)
public readonly requiresAuth = false; // API key is optional
public readonly rateLimit = {
requestsPerMinute: 30, // Demo plan default
requestsPerDay: 10000 // Demo plan monthly quota / 30
};
private logger = console;
private baseUrl = 'https://api.coingecko.com/api/v3';
private apiKey?: string;
private rateLimiter: RateLimiter;
// Coin mapping cache
private coinMapCache = new Map<string, string>(); // ticker/id -> coingecko id
private coinListLoadedAt: Date | null = null;
private readonly coinListCacheTTL = 24 * 60 * 60 * 1000; // 24 hours
// Priority map for common crypto tickers (to avoid conflicts)
private readonly priorityTickerMap = new Map<string, string>([
['btc', 'bitcoin'],
['eth', 'ethereum'],
['usdt', 'tether'],
['bnb', 'binancecoin'],
['sol', 'solana'],
['usdc', 'usd-coin'],
['xrp', 'ripple'],
['ada', 'cardano'],
['doge', 'dogecoin'],
['trx', 'tron'],
['dot', 'polkadot'],
['matic', 'matic-network'],
['ltc', 'litecoin'],
['shib', 'shiba-inu'],
['avax', 'avalanche-2'],
['link', 'chainlink'],
['atom', 'cosmos'],
['uni', 'uniswap'],
['etc', 'ethereum-classic'],
['xlm', 'stellar']
]);
constructor(apiKey?: string, private config?: IProviderConfig) {
this.apiKey = apiKey;
this.rateLimiter = new RateLimiter(this.rateLimit.requestsPerMinute);
}
/**
* Unified data fetching method supporting all request types
*/
public async fetchData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
switch (request.type) {
case 'current':
return this.fetchCurrentPrice(request);
case 'batch':
return this.fetchBatchCurrentPrices(request);
case 'historical':
return this.fetchHistoricalPrices(request);
case 'intraday':
return this.fetchIntradayPrices(request);
default:
throw new Error(`Unsupported request type: ${(request as any).type}`);
}
}
/**
* Fetch current price for a single crypto
*/
private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
return this.fetchWithRateLimitRetry(async () => {
// Resolve ticker to CoinGecko ID
const coinId = await this.resolveCoinId(request.ticker);
// Build URL
const params = new URLSearchParams({
ids: coinId,
vs_currencies: 'usd',
include_market_cap: 'true',
include_24hr_vol: 'true',
include_24hr_change: 'true',
include_last_updated_at: 'true'
});
const url = `${this.baseUrl}/simple/price?${params}`;
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Make request
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for rate limit error
if (this.isRateLimitError(responseData)) {
const waitTime = this.rateLimiter.handleRateLimitError();
throw new RateLimitError(
`Rate limit exceeded for ${request.ticker}`,
waitTime
);
}
if (!responseData[coinId]) {
throw new Error(`No data found for ${request.ticker} (${coinId})`);
}
return this.mapToStockPrice(request.ticker, coinId, responseData[coinId], 'live');
}, `current price for ${request.ticker}`);
}
/**
* Fetch batch current prices for multiple cryptos
*/
private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
return this.fetchWithRateLimitRetry(async () => {
// Resolve all tickers to CoinGecko IDs
const coinIds = await Promise.all(
request.tickers.map(ticker => this.resolveCoinId(ticker))
);
// Build URL with comma-separated IDs
const params = new URLSearchParams({
ids: coinIds.join(','),
vs_currencies: 'usd',
include_market_cap: 'true',
include_24hr_vol: 'true',
include_24hr_change: 'true',
include_last_updated_at: 'true'
});
const url = `${this.baseUrl}/simple/price?${params}`;
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Make request
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for rate limit error
if (this.isRateLimitError(responseData)) {
const waitTime = this.rateLimiter.handleRateLimitError();
throw new RateLimitError(
`Rate limit exceeded for batch request`,
waitTime
);
}
const prices: IStockPrice[] = [];
// Map responses back to original tickers
for (let i = 0; i < request.tickers.length; i++) {
const ticker = request.tickers[i];
const coinId = coinIds[i];
if (responseData[coinId]) {
try {
prices.push(this.mapToStockPrice(ticker, coinId, responseData[coinId], 'live'));
} catch (error) {
this.logger.warn(`Failed to parse data for ${ticker}:`, error);
}
} else {
this.logger.warn(`No data returned for ${ticker} (${coinId})`);
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch request');
}
return prices;
}, `batch prices for ${request.tickers.length} tickers`);
}
/**
* Fetch historical prices with OHLCV data
*/
private async fetchHistoricalPrices(request: IStockHistoricalRequest): Promise<IStockPrice[]> {
return this.fetchWithRateLimitRetry(async () => {
const coinId = await this.resolveCoinId(request.ticker);
// Calculate days between dates
const days = Math.ceil((request.to.getTime() - request.from.getTime()) / (1000 * 60 * 60 * 24));
// Build URL
const params = new URLSearchParams({
vs_currency: 'usd',
days: days.toString(),
interval: 'daily' // Explicit daily granularity for historical data
});
const url = `${this.baseUrl}/coins/${coinId}/market_chart?${params}`;
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Make request
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(this.config?.timeout || 20000)
.get();
const responseData = await response.json() as any;
// Check for rate limit error
if (this.isRateLimitError(responseData)) {
const waitTime = this.rateLimiter.handleRateLimitError();
throw new RateLimitError(
`Rate limit exceeded for historical ${request.ticker}`,
waitTime
);
}
if (!responseData.prices || !Array.isArray(responseData.prices)) {
this.logger.error(`Invalid API response for ${request.ticker}:`, JSON.stringify(responseData).substring(0, 500));
throw new Error(`Invalid response format for ${request.ticker}: ${JSON.stringify(responseData).substring(0, 200)}`);
}
const prices: IStockPrice[] = [];
const priceData = responseData.prices;
const marketCapData = responseData.market_caps || [];
const volumeData = responseData.total_volumes || [];
// Warn if processing large amount of historical data
const maxRecords = this.config?.maxRecords || 10000;
if (priceData.length > maxRecords) {
this.logger.warn(
`Historical request for ${request.ticker} returned ${priceData.length} records, ` +
`which exceeds maxRecords limit of ${maxRecords}. Processing first ${maxRecords} only.`
);
}
// Process each data point (up to maxRecords)
const recordsToProcess = Math.min(priceData.length, maxRecords);
for (let i = 0; i < recordsToProcess; i++) {
const [timestamp, price] = priceData[i];
const date = new Date(timestamp);
// Filter by date range
if (date < request.from || date > request.to) continue;
const marketCap = marketCapData[i]?.[1];
const volume = volumeData[i]?.[1];
// Calculate previous close for change calculation
const previousClose = i > 0 ? priceData[i - 1][1] : price;
const change = price - previousClose;
const changePercent = previousClose !== 0 ? (change / previousClose) * 100 : 0;
prices.push({
ticker: request.ticker.toUpperCase(),
price: price,
currency: 'USD',
change: change,
changePercent: changePercent,
previousClose: previousClose,
timestamp: date,
provider: this.name,
marketState: 'REGULAR', // Crypto markets are always open
// OHLCV data (note: market_chart doesn't provide OHLC, only close prices)
volume: volume,
dataType: 'eod',
fetchedAt: new Date(),
companyName: coinId.charAt(0).toUpperCase() + coinId.slice(1)
});
}
return prices;
}, `historical prices for ${request.ticker}`);
}
/**
* Fetch intraday prices with hourly intervals
*/
private async fetchIntradayPrices(request: IStockIntradayRequest): Promise<IStockPrice[]> {
return this.fetchWithRateLimitRetry(async () => {
const coinId = await this.resolveCoinId(request.ticker);
// Map interval to days parameter (CoinGecko auto-granularity)
// For hourly data, request 1-7 days
let days = 1;
switch (request.interval) {
case '1min':
case '5min':
case '10min':
case '15min':
case '30min':
throw new Error('CoinGecko only supports hourly intervals in market_chart. Use interval: "1hour"');
case '1hour':
days = 1; // Last 24 hours with hourly granularity
break;
}
// Build URL (omit interval param for automatic granularity based on days)
const params = new URLSearchParams({
vs_currency: 'usd',
days: days.toString()
});
const url = `${this.baseUrl}/coins/${coinId}/market_chart?${params}`;
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Make request
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for rate limit error
if (this.isRateLimitError(responseData)) {
const waitTime = this.rateLimiter.handleRateLimitError();
throw new RateLimitError(
`Rate limit exceeded for intraday ${request.ticker}`,
waitTime
);
}
if (!responseData.prices || !Array.isArray(responseData.prices)) {
this.logger.error(`Invalid API response for ${request.ticker}:`, JSON.stringify(responseData).substring(0, 500));
throw new Error(`Invalid response format for ${request.ticker}: ${JSON.stringify(responseData).substring(0, 200)}`);
}
const prices: IStockPrice[] = [];
const priceData = responseData.prices;
const marketCapData = responseData.market_caps || [];
const volumeData = responseData.total_volumes || [];
// Apply default limit if user didn't specify one (performance optimization)
const effectiveLimit = request.limit || this.config?.defaultIntradayLimit || 1000;
// Warn if fetching large amount of data without explicit limit
if (!request.limit && priceData.length > effectiveLimit) {
this.logger.warn(
`Intraday request for ${request.ticker} returned ${priceData.length} records but no limit specified. ` +
`Applying default limit of ${effectiveLimit}. Consider adding a limit to the request for better performance.`
);
}
// Apply limit (take most recent data)
const limit = Math.min(effectiveLimit, priceData.length);
const dataToProcess = priceData.slice(-limit);
for (let i = 0; i < dataToProcess.length; i++) {
const actualIndex = priceData.length - limit + i;
const [timestamp, price] = dataToProcess[i];
const date = new Date(timestamp);
const marketCap = marketCapData[actualIndex]?.[1];
const volume = volumeData[actualIndex]?.[1];
const previousClose = i > 0 ? dataToProcess[i - 1][1] : price;
const change = price - previousClose;
const changePercent = previousClose !== 0 ? (change / previousClose) * 100 : 0;
prices.push({
ticker: request.ticker.toUpperCase(),
price: price,
currency: 'USD',
change: change,
changePercent: changePercent,
previousClose: previousClose,
timestamp: date,
provider: this.name,
marketState: 'REGULAR',
volume: volume,
dataType: 'intraday',
fetchedAt: new Date(),
companyName: coinId.charAt(0).toUpperCase() + coinId.slice(1)
});
}
return prices;
}, `intraday prices for ${request.ticker}`);
}
/**
* Check if CoinGecko API is available
*/
public async isAvailable(): Promise<boolean> {
try {
const url = `${this.baseUrl}/simple/price?ids=bitcoin&vs_currencies=usd`;
await this.rateLimiter.waitForSlot();
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(5000)
.get();
const responseData = await response.json() as any;
return responseData.bitcoin?.usd !== undefined;
} catch (error) {
this.logger.warn('CoinGecko provider is not available:', error);
return false;
}
}
/**
* Check if a market/network is supported
* CoinGecko supports 240+ networks
*/
public supportsMarket(market: string): boolean {
// CoinGecko has extensive crypto network coverage
const supportedNetworks = [
'CRYPTO', 'BTC', 'ETH', 'BSC', 'POLYGON', 'AVALANCHE',
'SOLANA', 'ARBITRUM', 'OPTIMISM', 'BASE'
];
return supportedNetworks.includes(market.toUpperCase());
}
/**
* Check if a ticker format is supported
* Supports both ticker symbols (BTC) and CoinGecko IDs (bitcoin)
*/
public supportsTicker(ticker: string): boolean {
// Accept alphanumeric with hyphens (for coin IDs like 'wrapped-bitcoin')
return /^[A-Za-z0-9\-]{1,50}$/.test(ticker);
}
/**
* Resolve ticker symbol or CoinGecko ID to canonical CoinGecko ID
* Supports both formats: "BTC" -> "bitcoin", "bitcoin" -> "bitcoin"
*/
private async resolveCoinId(tickerOrId: string): Promise<string> {
const normalized = tickerOrId.toLowerCase();
// Check priority map first (for common cryptos)
if (this.priorityTickerMap.has(normalized)) {
const coinId = this.priorityTickerMap.get(normalized)!;
this.coinMapCache.set(normalized, coinId);
return coinId;
}
// Check cache
if (this.coinMapCache.has(normalized)) {
return this.coinMapCache.get(normalized)!;
}
// Check if it's already a valid CoinGecko ID (contains hyphens or is all lowercase with original case)
if (normalized.includes('-') || normalized === tickerOrId) {
// Assume it's a CoinGecko ID, cache it
this.coinMapCache.set(normalized, normalized);
return normalized;
}
// Load coin list if needed
if (!this.coinListLoadedAt ||
Date.now() - this.coinListLoadedAt.getTime() > this.coinListCacheTTL) {
await this.loadCoinList();
}
// Try to find in cache after loading
if (this.coinMapCache.has(normalized)) {
return this.coinMapCache.get(normalized)!;
}
// Not found - return as-is and let API handle the error
this.logger.warn(`Could not resolve ticker ${tickerOrId} to CoinGecko ID, using as-is`);
return normalized;
}
/**
* Load complete coin list from CoinGecko API
*/
private async loadCoinList(): Promise<void> {
try {
const url = `${this.baseUrl}/coins/list`;
await this.rateLimiter.waitForSlot();
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers(this.buildHeaders())
.timeout(10000)
.get();
const coinList = await response.json() as ICoinListItem[];
// Clear cache before rebuilding to prevent memory leak
// Keep only entries that are in priorityTickerMap
const priorityEntries = new Map<string, string>();
for (const [key, value] of this.priorityTickerMap) {
priorityEntries.set(key, value);
}
this.coinMapCache.clear();
// Restore priority mappings
for (const [key, value] of priorityEntries) {
this.coinMapCache.set(key, value);
}
// Build mapping: symbol -> id
for (const coin of coinList) {
const symbol = coin.symbol.toLowerCase();
const id = coin.id.toLowerCase();
// Don't overwrite priority mappings
if (!this.priorityTickerMap.has(symbol)) {
this.coinMapCache.set(symbol, id);
}
// Always cache the ID mapping (id -> id for when users pass CoinGecko IDs directly)
this.coinMapCache.set(id, id);
}
this.coinListLoadedAt = new Date();
this.logger.info(`Loaded ${coinList.length} coins from CoinGecko (cache: ${this.coinMapCache.size} entries)`);
} catch (error) {
this.logger.error('Failed to load coin list from CoinGecko:', error);
// Don't throw - we can still work with direct IDs
}
}
/**
* Map CoinGecko simple/price response to IStockPrice
*/
private mapToStockPrice(
ticker: string,
coinId: string,
data: any,
dataType: 'live' | 'eod' | 'intraday'
): IStockPrice {
const price = data.usd;
const change24h = data.usd_24h_change || 0;
// Calculate previous close from 24h change
const changePercent = change24h;
const change = (price * changePercent) / 100;
const previousClose = price - change;
// Parse last updated timestamp
const timestamp = data.last_updated_at
? new Date(data.last_updated_at * 1000)
: new Date();
return {
ticker: ticker.toUpperCase(),
price: price,
currency: 'USD',
change: change,
changePercent: changePercent,
previousClose: previousClose,
timestamp: timestamp,
provider: this.name,
marketState: 'REGULAR', // Crypto markets are 24/7
// Volume and market cap
volume: data.usd_24h_vol,
dataType: dataType,
fetchedAt: new Date(),
// Company identification (use coin name)
companyName: coinId.charAt(0).toUpperCase() + coinId.slice(1),
companyFullName: `${coinId.charAt(0).toUpperCase() + coinId.slice(1)} (${ticker.toUpperCase()})`
};
}
/**
* Build HTTP headers with optional API key
*/
private buildHeaders(): Record<string, string> {
const headers: Record<string, string> = {
'Accept': 'application/json'
};
if (this.apiKey) {
// Use Demo or Pro API key header
// CoinGecko accepts both x-cg-demo-api-key and x-cg-pro-api-key
headers['x-cg-demo-api-key'] = this.apiKey;
}
return headers;
}
/**
* Check if response indicates a rate limit error (429)
*/
private isRateLimitError(responseData: any): boolean {
return responseData?.status?.error_code === 429;
}
/**
* Wrapper for fetch operations with automatic rate limit retry and exponential backoff
*/
private async fetchWithRateLimitRetry<T>(
fetchFn: () => Promise<T>,
operationName: string,
maxRetries: number = 3
): Promise<T> {
let lastError: Error | undefined;
for (let attempt = 0; attempt < maxRetries; attempt++) {
try {
const result = await fetchFn();
this.rateLimiter.resetErrors();
return result;
} catch (error) {
lastError = error as Error;
if (error instanceof RateLimitError) {
const attemptInfo = `${attempt + 1}/${maxRetries}`;
this.logger.warn(
`Rate limit hit for ${operationName}, waiting ${error.waitTime}ms before retry ${attemptInfo}`
);
if (attempt < maxRetries - 1) {
await plugins.smartdelay.delayFor(error.waitTime);
continue;
} else {
this.logger.error(`Max retries (${maxRetries}) exceeded for ${operationName} due to rate limiting`);
throw error;
}
}
// Non-rate-limit errors: throw immediately
throw error;
}
}
throw lastError!;
}
}

View File

@@ -0,0 +1,704 @@
import * as plugins from '../../plugins.js';
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
import type {
IStockPrice,
IStockDataRequest,
IStockCurrentRequest,
IStockHistoricalRequest,
IStockIntradayRequest,
IStockBatchCurrentRequest
} from '../interfaces/stockprice.js';
/**
* Marketstack API v2 Provider - Professional Plan with Intelligent Intraday Support
* Documentation: https://docs.apilayer.com/marketstack/docs/marketstack-api-v2-v-2-0-0
*
* Features:
* - Intelligent endpoint selection based on market hours
* - Real-time intraday pricing with multiple intervals (1min, 5min, 10min, 15min, 30min, 1hour)
* - End-of-Day (EOD) stock prices with historical data
* - Market state detection (PRE, REGULAR, POST, CLOSED)
* - Exchange filtering via MIC codes (XNAS, XNYS, XLON, etc.)
* - Supports 500,000+ tickers across 72+ exchanges worldwide
* - OHLCV data (Open, High, Low, Close, Volume)
* - Pagination for large datasets
* - Automatic fallback from intraday to EOD on errors
* - Requires API key authentication
*
* Intelligent Endpoint Selection:
* - During market hours (PRE/REGULAR/POST): Uses intraday endpoints for fresh data
* - After market close (CLOSED): Uses EOD endpoints to save API credits
* - Automatic fallback to EOD if intraday fails (rate limits, plan restrictions, etc.)
*
* Rate Limits:
* - Free Plan: 100 requests/month (EOD only)
* - Basic Plan: 10,000 requests/month (EOD only)
* - Professional Plan: 100,000 requests/month (intraday + EOD)
*
* Intraday Access:
* - Intervals below 15min (1min, 5min, 10min) require Professional Plan or higher
* - Real-time data from IEX Exchange for US tickers
* - Symbol formatting: Periods replaced with hyphens for intraday (BRK.B → BRK-B)
*/
export class MarketstackProvider implements IStockProvider {
public name = 'Marketstack';
public priority = 90; // Increased from 80 - now supports real-time intraday data during market hours
public readonly requiresAuth = true;
public readonly rateLimit = {
requestsPerMinute: undefined, // No per-minute limit specified
requestsPerDay: undefined // Varies by plan
};
private logger = console;
private baseUrl = 'https://api.marketstack.com/v2';
private apiKey: string;
constructor(apiKey: string, private config?: IProviderConfig) {
if (!apiKey) {
throw new Error('API key is required for Marketstack provider');
}
this.apiKey = apiKey;
}
/**
* Unified data fetching method supporting all request types
*/
public async fetchData(request: IStockDataRequest): Promise<IStockPrice[] | IStockPrice> {
switch (request.type) {
case 'current':
return this.fetchCurrentPrice(request);
case 'historical':
return this.fetchHistoricalPrices(request);
case 'intraday':
return this.fetchIntradayPrices(request);
case 'batch':
return this.fetchBatchCurrentPrices(request);
default:
throw new Error(`Unsupported request type: ${(request as any).type}`);
}
}
/**
* Fetch current/latest price with intelligent endpoint selection
* Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
* Uses EOD after market close (CLOSED) to save API credits
*/
private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
try {
// Determine current market state
const marketState = this.getUsMarketState();
const useIntraday = this.shouldUseIntradayEndpoint(marketState);
if (useIntraday) {
// Use intraday endpoint for fresh data during market hours
return await this.fetchCurrentPriceIntraday(request, marketState);
} else {
// Use EOD endpoint for after-close data
return await this.fetchCurrentPriceEod(request);
}
} catch (error) {
// If intraday fails, fallback to EOD with warning
if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
this.logger.warn(`Intraday endpoint failed for ${request.ticker}, falling back to EOD:`, error.message);
try {
return await this.fetchCurrentPriceEod(request);
} catch (eodError) {
// Both failed, throw original error
throw error;
}
}
throw error;
}
}
/**
* Fetch current price using intraday endpoint (during market hours)
* Uses 1min interval for most recent data
*/
private async fetchCurrentPriceIntraday(
request: IStockCurrentRequest,
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): Promise<IStockPrice> {
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday/latest?access_key=${this.apiKey}`;
url += `&interval=1min`; // Use 1min for most recent data
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData || !responseData.close) {
throw new Error(`No intraday data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData, 'intraday', marketState);
}
/**
* Fetch current price using EOD endpoint (after market close)
*/
private async fetchCurrentPriceEod(request: IStockCurrentRequest): Promise<IStockPrice> {
let url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
// For single ticker endpoint, response is direct object (not wrapped in data field)
if (!responseData || !responseData.close) {
throw new Error(`No data found for ticker ${request.ticker}`);
}
return this.mapToStockPrice(responseData, 'eod');
}
/**
* Fetch historical EOD prices for a ticker with date range
*/
private async fetchHistoricalPrices(request: IStockHistoricalRequest): Promise<IStockPrice[]> {
try {
const allPrices: IStockPrice[] = [];
let offset = request.offset || 0;
const limit = request.limit || 1000; // Max per page
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
while (true) {
let url = `${this.baseUrl}/eod?access_key=${this.apiKey}`;
url += `&symbols=${request.ticker}`;
url += `&date_from=${this.formatDate(request.from)}`;
url += `&date_to=${this.formatDate(request.to)}`;
url += `&limit=${limit}`;
url += `&offset=${offset}`;
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
if (request.sort) {
url += `&sort=${request.sort}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
// Map data to stock prices
for (const data of responseData.data) {
try {
allPrices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse historical data for ${data.symbol}:`, error);
}
}
// Check if we have more pages
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Safety check: don't fetch more than maxRecords
if (!hasMore || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
return allPrices;
} catch (error) {
this.logger.error(`Failed to fetch historical prices for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch historical prices for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch intraday prices with specified interval
* Supports intervals: 1min, 5min, 10min, 15min, 30min, 1hour
* Note: Intervals below 15min require Professional Plan or higher
*/
private async fetchIntradayPrices(request: IStockIntradayRequest): Promise<IStockPrice[]> {
try {
const allPrices: IStockPrice[] = [];
let offset = 0;
const limit = 1000; // Max per page for intraday
const maxRecords = this.config?.maxRecords || 10000; // Safety limit (configurable)
// Apply default limit if user didn't specify one (performance optimization)
const effectiveLimit = request.limit || this.config?.defaultIntradayLimit || 1000;
// Warn if fetching large amount of data without explicit limit
if (!request.limit && effectiveLimit > 1000) {
this.logger.warn(
`Intraday request for ${request.ticker} without explicit limit will fetch up to ${effectiveLimit} records. ` +
`Consider adding a limit to the request for better performance.`
);
}
// Format symbol for intraday endpoint (replace . with -)
const formattedSymbol = this.formatSymbolForIntraday(request.ticker);
while (true) {
let url = `${this.baseUrl}/tickers/${formattedSymbol}/intraday?access_key=${this.apiKey}`;
url += `&interval=${request.interval}`;
url += `&limit=${limit}`;
url += `&offset=${offset}`;
// Add date filter if specified
if (request.date) {
url += `&date_from=${this.formatDate(request.date)}`;
url += `&date_to=${this.formatDate(request.date)}`;
}
// Add exchange filter if specified
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
// Map data to stock prices
for (const data of responseData.data) {
try {
allPrices.push(this.mapToStockPrice(data, 'intraday'));
} catch (error) {
this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
}
}
// Check if we have more pages
const pagination = responseData.pagination;
const hasMore = pagination && offset + limit < pagination.total;
// Honor effective limit or safety maxRecords
if (!hasMore || allPrices.length >= effectiveLimit || allPrices.length >= maxRecords) {
break;
}
offset += limit;
}
// Apply effective limit
if (allPrices.length > effectiveLimit) {
return allPrices.slice(0, effectiveLimit);
}
return allPrices;
} catch (error) {
this.logger.error(`Failed to fetch intraday prices for ${request.ticker}:`, error);
throw new Error(`Marketstack: Failed to fetch intraday prices for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch current prices for multiple tickers with intelligent endpoint selection
* Uses intraday during market hours (PRE, REGULAR, POST) for fresh data
* Uses EOD after market close (CLOSED) to save API credits
*/
private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
try {
// Determine current market state
const marketState = this.getUsMarketState();
const useIntraday = this.shouldUseIntradayEndpoint(marketState);
if (useIntraday) {
return await this.fetchBatchCurrentPricesIntraday(request, marketState);
} else {
return await this.fetchBatchCurrentPricesEod(request);
}
} catch (error) {
// Fallback to EOD if intraday fails
if (error.message?.includes('intraday') || error.message?.includes('Marketstack API error')) {
this.logger.warn(`Intraday batch endpoint failed, falling back to EOD:`, error.message);
try {
return await this.fetchBatchCurrentPricesEod(request);
} catch (eodError) {
// Both failed, throw original error
throw error;
}
}
throw error;
}
}
/**
* Fetch batch current prices using intraday endpoint
*/
private async fetchBatchCurrentPricesIntraday(
request: IStockBatchCurrentRequest,
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): Promise<IStockPrice[]> {
// Format symbols for intraday (replace . with -)
const formattedSymbols = request.tickers.map(t => this.formatSymbolForIntraday(t)).join(',');
let url = `${this.baseUrl}/intraday/latest?access_key=${this.apiKey}`;
url += `&symbols=${formattedSymbols}`;
url += `&interval=1min`; // Use 1min for most recent data
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
const prices: IStockPrice[] = [];
for (const data of responseData.data) {
try {
prices.push(this.mapToStockPrice(data, 'intraday', marketState));
} catch (error) {
this.logger.warn(`Failed to parse intraday data for ${data.symbol}:`, error);
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch intraday request');
}
return prices;
}
/**
* Fetch batch current prices using EOD endpoint
*/
private async fetchBatchCurrentPricesEod(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
const symbols = request.tickers.join(',');
let url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
if (request.exchange) {
url += `&exchange=${request.exchange}`;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
// Check for API errors
if (responseData.error) {
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
}
if (!responseData?.data || !Array.isArray(responseData.data)) {
throw new Error('Invalid response format from Marketstack API');
}
const prices: IStockPrice[] = [];
for (const data of responseData.data) {
try {
prices.push(this.mapToStockPrice(data, 'eod'));
} catch (error) {
this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
// Continue processing other tickers
}
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch request');
}
return prices;
}
/**
* Check if the Marketstack API is available and accessible
*/
public async isAvailable(): Promise<boolean> {
try {
// Test with a well-known ticker
const url = `${this.baseUrl}/tickers/AAPL/eod/latest?access_key=${this.apiKey}`;
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.timeout(5000)
.get();
const responseData = await response.json() as any;
// Check if we got valid data (not an error)
// Single ticker endpoint returns direct object, not wrapped in data field
return !responseData.error && responseData.close !== undefined;
} catch (error) {
this.logger.warn('Marketstack provider is not available:', error);
return false;
}
}
/**
* Check if a market is supported
* Marketstack supports 72+ exchanges worldwide
*/
public supportsMarket(market: string): boolean {
// Marketstack has broad international coverage including:
// US, UK, DE, FR, JP, CN, HK, AU, CA, IN, etc.
const supportedMarkets = [
'US', 'UK', 'GB', 'DE', 'FR', 'JP', 'CN', 'HK', 'AU', 'CA',
'IN', 'BR', 'MX', 'IT', 'ES', 'NL', 'SE', 'CH', 'NO', 'DK'
];
return supportedMarkets.includes(market.toUpperCase());
}
/**
* Check if a ticker format is supported
*/
public supportsTicker(ticker: string): boolean {
// Basic validation - Marketstack supports most standard ticker formats
return /^[A-Z0-9\.\-]{1,10}$/.test(ticker.toUpperCase());
}
/**
* Map Marketstack API response to IStockPrice interface
* @param data - API response data
* @param dataType - Type of data (eod, intraday, live)
* @param explicitMarketState - Override market state (used for intraday data fetched during known market hours)
*/
private mapToStockPrice(
data: any,
dataType: 'eod' | 'intraday' | 'live' = 'eod',
explicitMarketState?: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'
): IStockPrice {
if (!data.close) {
throw new Error('Missing required price data');
}
// Calculate change and change percent
// EOD data: previous close is typically open price of the same day
// For better accuracy, we'd need previous day's close, but that requires another API call
const currentPrice = data.close;
const previousClose = data.open || currentPrice;
const change = currentPrice - previousClose;
const changePercent = previousClose !== 0 ? (change / previousClose) * 100 : 0;
// Parse timestamp
const timestamp = data.date ? new Date(data.date) : new Date();
const fetchedAt = new Date();
// Determine market state intelligently
let marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
if (explicitMarketState) {
// Use provided market state (for intraday data fetched during known market hours)
marketState = explicitMarketState;
} else if (dataType === 'eod') {
// EOD data is always for closed markets
marketState = 'CLOSED';
} else if (dataType === 'intraday') {
// For intraday data without explicit state, determine from timestamp
marketState = this.getUsMarketState(timestamp.getTime());
} else {
// Default fallback
marketState = 'CLOSED';
}
const stockPrice: IStockPrice = {
ticker: data.symbol.toUpperCase(),
price: currentPrice,
currency: data.price_currency || 'USD',
change: change,
changePercent: changePercent,
previousClose: previousClose,
timestamp: timestamp,
provider: this.name,
marketState: marketState, // Now dynamic based on data type, timestamp, and explicit state
exchange: data.exchange,
exchangeName: data.exchange_code || data.name,
// Phase 1 enhancements: OHLCV data
volume: data.volume,
open: data.open,
high: data.high,
low: data.low,
adjusted: data.adj_close !== undefined, // If adj_close exists, price is adjusted
dataType: dataType,
fetchedAt: fetchedAt,
// Company identification
companyName: data.company_name || data.name || undefined,
companyFullName: this.buildCompanyFullName(data)
};
return stockPrice;
}
/**
* Build full company name with exchange and ticker information
* Example: "Apple Inc (NASDAQ:AAPL)"
*/
private buildCompanyFullName(data: any): string | undefined {
// Check if API already provides full name
if (data.full_name || data.long_name) {
return data.full_name || data.long_name;
}
// Build from available data
const companyName = data.company_name || data.name;
const exchangeCode = data.exchange_code; // e.g., "NASDAQ"
const symbol = data.symbol; // e.g., "AAPL"
if (!companyName) {
return undefined;
}
// If we have exchange and symbol, build full name: "Apple Inc (NASDAQ:AAPL)"
if (exchangeCode && symbol) {
return `${companyName} (${exchangeCode}:${symbol})`;
}
// If we only have symbol: "Apple Inc (AAPL)"
if (symbol) {
return `${companyName} (${symbol})`;
}
// Otherwise just return company name
return companyName;
}
/**
* Format date to YYYY-MM-DD for API requests
*/
private formatDate(date: Date): string {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0');
const day = String(date.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
/**
* Get US market state based on Eastern Time
* Regular hours: 9:30 AM - 4:00 PM ET
* Pre-market: 4:00 AM - 9:30 AM ET
* After-hours: 4:00 PM - 8:00 PM ET
*
* @param timestampMs - Optional timestamp in milliseconds (defaults to current time)
* @returns Market state: PRE, REGULAR, POST, or CLOSED
*/
private getUsMarketState(timestampMs?: number): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' {
const now = timestampMs ? new Date(timestampMs) : new Date();
// Convert to ET (UTC-5 or UTC-4 depending on DST)
// For simplicity, we'll use a rough approximation
// TODO: Add proper timezone library for production use
const etOffset = -5; // Standard time, adjust for DST if needed
const etTime = new Date(now.getTime() + (etOffset * 60 * 60 * 1000));
// Get day of week (0 = Sunday, 6 = Saturday)
const dayOfWeek = etTime.getUTCDay();
// Check if weekend
if (dayOfWeek === 0 || dayOfWeek === 6) {
return 'CLOSED';
}
// Get hour and minute in ET
const hours = etTime.getUTCHours();
const minutes = etTime.getUTCMinutes();
const timeInMinutes = hours * 60 + minutes;
// Define market hours in minutes
const preMarketStart = 4 * 60; // 4:00 AM
const regularMarketStart = 9 * 60 + 30; // 9:30 AM
const regularMarketEnd = 16 * 60; // 4:00 PM
const afterHoursEnd = 20 * 60; // 8:00 PM
if (timeInMinutes >= preMarketStart && timeInMinutes < regularMarketStart) {
return 'PRE';
} else if (timeInMinutes >= regularMarketStart && timeInMinutes < regularMarketEnd) {
return 'REGULAR';
} else if (timeInMinutes >= regularMarketEnd && timeInMinutes < afterHoursEnd) {
return 'POST';
} else {
return 'CLOSED';
}
}
/**
* Determine if intraday endpoint should be used based on market state
* Uses intraday for PRE, REGULAR, and POST market states
* Uses EOD for CLOSED state to save API credits
*
* @param marketState - Current market state
* @returns true if intraday endpoint should be used
*/
private shouldUseIntradayEndpoint(marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED'): boolean {
return marketState !== 'CLOSED';
}
/**
* Format ticker symbol for intraday endpoints
* Marketstack intraday API requires periods to be replaced with hyphens
* Example: BRK.B → BRK-B
*
* @param symbol - Original ticker symbol
* @returns Formatted symbol for intraday endpoints
*/
private formatSymbolForIntraday(symbol: string): string {
return symbol.replace(/\./g, '-');
}
}

View File

@@ -0,0 +1,467 @@
import * as plugins from '../../plugins.js';
import type {
IFundamentalsProvider,
IStockFundamentals,
IFundamentalsRequest,
IFundamentalsCurrentRequest,
IFundamentalsBatchRequest
} from '../interfaces/fundamentals.js';
/**
* Configuration for SEC EDGAR provider
*/
export interface ISecEdgarConfig {
userAgent: string; // Required: Format "Company Name Email" (e.g., "fin.cx info@fin.cx")
cikCacheTTL?: number; // Default: 30 days (CIK codes rarely change)
fundamentalsCacheTTL?: number; // Default: 90 days (quarterly filings)
timeout?: number; // Request timeout in ms
}
/**
* Rate limiter for SEC EDGAR API
* SEC requires: 10 requests per second maximum
*/
class RateLimiter {
private requestTimes: number[] = [];
private readonly maxRequestsPerSecond = 10;
public async waitForSlot(): Promise<void> {
const now = Date.now();
const oneSecondAgo = now - 1000;
// Remove requests older than 1 second
this.requestTimes = this.requestTimes.filter(time => time > oneSecondAgo);
// If we've hit the limit, wait
if (this.requestTimes.length >= this.maxRequestsPerSecond) {
const oldestRequest = this.requestTimes[0];
const waitTime = 1000 - (now - oldestRequest) + 10; // +10ms buffer
await plugins.smartdelay.delayFor(waitTime);
return this.waitForSlot(); // Recursively check again
}
// Record this request
this.requestTimes.push(now);
}
}
/**
* SEC EDGAR Fundamental Data Provider
*
* Features:
* - Free public access (no API key required)
* - All US public companies
* - Financial data from 10-K/10-Q filings
* - US GAAP standardized metrics
* - Historical data back to ~2009
* - < 1 minute filing delay
*
* Documentation: https://www.sec.gov/edgar/sec-api-documentation
*
* Rate Limits:
* - 10 requests per second (enforced by SEC)
* - Requires User-Agent header in format: "Company Name Email"
*
* Data Sources:
* - Company Facts API: /api/xbrl/companyfacts/CIK##########.json
* - Ticker Lookup: /files/company_tickers.json
*/
export class SecEdgarProvider implements IFundamentalsProvider {
public name = 'SEC EDGAR';
public priority = 100; // High priority - free, authoritative, comprehensive
public readonly requiresAuth = false; // No API key needed!
public readonly rateLimit = {
requestsPerMinute: 600, // 10 requests/second = 600/minute
requestsPerDay: undefined // No daily limit
};
private logger = console;
private baseUrl = 'https://data.sec.gov/api/xbrl';
private tickersUrl = 'https://www.sec.gov/files/company_tickers.json';
private userAgent: string;
private config: Required<ISecEdgarConfig>;
// Caching
private cikCache = new Map<string, { cik: string; timestamp: Date }>();
private tickerListCache: { data: any; timestamp: Date } | null = null;
// Rate limiting
private rateLimiter = new RateLimiter();
constructor(config: ISecEdgarConfig) {
// Validate User-Agent
if (!config.userAgent) {
throw new Error('User-Agent is required for SEC EDGAR provider (format: "Company Name Email")');
}
// Validate User-Agent format (must contain space and @ symbol)
if (!config.userAgent.includes(' ') || !config.userAgent.includes('@')) {
throw new Error(
'Invalid User-Agent format. Required: "Company Name Email" (e.g., "fin.cx info@fin.cx")'
);
}
this.userAgent = config.userAgent;
this.config = {
userAgent: config.userAgent,
cikCacheTTL: config.cikCacheTTL || 30 * 24 * 60 * 60 * 1000, // 30 days
fundamentalsCacheTTL: config.fundamentalsCacheTTL || 90 * 24 * 60 * 60 * 1000, // 90 days
timeout: config.timeout || 30000
};
}
/**
* Unified data fetching method
*/
public async fetchData(
request: IFundamentalsRequest
): Promise<IStockFundamentals | IStockFundamentals[]> {
switch (request.type) {
case 'fundamentals-current':
return this.fetchFundamentals(request);
case 'fundamentals-batch':
return this.fetchBatchFundamentals(request);
default:
throw new Error(`Unsupported request type: ${(request as any).type}`);
}
}
/**
* Fetch fundamental data for a single ticker
*/
private async fetchFundamentals(request: IFundamentalsCurrentRequest): Promise<IStockFundamentals> {
try {
// 1. Get CIK for ticker (with caching)
const cik = await this.getCIK(request.ticker);
// 2. Fetch company facts from SEC (with rate limiting)
const companyFacts = await this.fetchCompanyFacts(cik);
// 3. Parse facts into structured fundamental data
return this.parseCompanyFacts(request.ticker, cik, companyFacts);
} catch (error) {
this.logger.error(`Failed to fetch fundamentals for ${request.ticker}:`, error);
throw new Error(`SEC EDGAR: Failed to fetch fundamentals for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch fundamental data for multiple tickers
*/
private async fetchBatchFundamentals(
request: IFundamentalsBatchRequest
): Promise<IStockFundamentals[]> {
const results: IStockFundamentals[] = [];
const errors: string[] = [];
for (const ticker of request.tickers) {
try {
const fundamentals = await this.fetchFundamentals({
type: 'fundamentals-current',
ticker
});
results.push(fundamentals);
} catch (error) {
this.logger.warn(`Failed to fetch fundamentals for ${ticker}:`, error);
errors.push(`${ticker}: ${error.message}`);
// Continue with other tickers
}
}
if (results.length === 0) {
throw new Error(`Failed to fetch fundamentals for all tickers. Errors: ${errors.join(', ')}`);
}
return results;
}
/**
* Get CIK (Central Index Key) for a ticker symbol
* Uses SEC's public ticker-to-CIK mapping file
*/
private async getCIK(ticker: string): Promise<string> {
const tickerUpper = ticker.toUpperCase();
// Check cache first
const cached = this.cikCache.get(tickerUpper);
if (cached) {
const age = Date.now() - cached.timestamp.getTime();
if (age < this.config.cikCacheTTL) {
return cached.cik;
}
// Cache expired, remove it
this.cikCache.delete(tickerUpper);
}
// Fetch ticker list (with caching at list level)
const tickers = await this.fetchTickerList();
// Find ticker in list (case-insensitive)
const entry = Object.values(tickers).find((t: any) => t.ticker === tickerUpper);
if (!entry) {
throw new Error(`CIK not found for ticker ${ticker}`);
}
const cik = String((entry as any).cik_str);
// Cache the result
this.cikCache.set(tickerUpper, {
cik,
timestamp: new Date()
});
return cik;
}
/**
* Fetch the SEC ticker-to-CIK mapping list
* Cached for 24 hours (list updates daily)
* Uses native fetch for automatic gzip decompression
*/
private async fetchTickerList(): Promise<any> {
// Check cache
if (this.tickerListCache) {
const age = Date.now() - this.tickerListCache.timestamp.getTime();
if (age < 24 * 60 * 60 * 1000) { // 24 hours
return this.tickerListCache.data;
}
}
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Fetch from SEC using native fetch (handles gzip automatically)
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
try {
const response = await fetch(this.tickersUrl, {
headers: {
'User-Agent': this.userAgent,
'Accept': 'application/json'
// Note: Accept-Encoding is set automatically by fetch
},
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const data = await response.json();
// Cache the list
this.tickerListCache = {
data,
timestamp: new Date()
};
return data;
} catch (error) {
clearTimeout(timeoutId);
throw error;
}
}
/**
* Fetch company facts from SEC EDGAR
* Uses native fetch for automatic gzip decompression
*/
private async fetchCompanyFacts(cik: string): Promise<any> {
// Pad CIK to 10 digits
const paddedCIK = cik.padStart(10, '0');
const url = `${this.baseUrl}/companyfacts/CIK${paddedCIK}.json`;
// Wait for rate limit slot
await this.rateLimiter.waitForSlot();
// Fetch from SEC using native fetch (handles gzip automatically)
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
try {
const response = await fetch(url, {
headers: {
'User-Agent': this.userAgent,
'Accept': 'application/json',
'Host': 'data.sec.gov'
// Note: Accept-Encoding is set automatically by fetch and gzip is handled transparently
},
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const data = await response.json();
// Validate response
if (!data || !data.facts) {
throw new Error('Invalid response from SEC EDGAR API');
}
return data;
} catch (error) {
clearTimeout(timeoutId);
throw error;
}
}
/**
* Parse SEC company facts into structured fundamental data
*/
private parseCompanyFacts(ticker: string, cik: string, data: any): IStockFundamentals {
const usGaap = data.facts?.['us-gaap'];
if (!usGaap) {
throw new Error('No US GAAP data available');
}
// Extract latest values for key metrics
const fundamentals: IStockFundamentals = {
ticker: ticker.toUpperCase(),
cik: cik,
companyName: data.entityName,
provider: this.name,
timestamp: new Date(),
fetchedAt: new Date(),
// Per-share metrics
earningsPerShareBasic: this.getLatestValue(usGaap, 'EarningsPerShareBasic'),
earningsPerShareDiluted: this.getLatestValue(usGaap, 'EarningsPerShareDiluted'),
sharesOutstanding: this.getLatestValue(usGaap, 'CommonStockSharesOutstanding'),
weightedAverageSharesOutstanding: this.getLatestValue(
usGaap,
'WeightedAverageNumberOfSharesOutstandingBasic'
),
// Income statement
revenue: this.getLatestValue(usGaap, 'Revenues') ||
this.getLatestValue(usGaap, 'RevenueFromContractWithCustomerExcludingAssessedTax'),
netIncome: this.getLatestValue(usGaap, 'NetIncomeLoss'),
operatingIncome: this.getLatestValue(usGaap, 'OperatingIncomeLoss'),
grossProfit: this.getLatestValue(usGaap, 'GrossProfit'),
costOfRevenue: this.getLatestValue(usGaap, 'CostOfRevenue'),
// Balance sheet
assets: this.getLatestValue(usGaap, 'Assets'),
liabilities: this.getLatestValue(usGaap, 'Liabilities'),
stockholdersEquity: this.getLatestValue(usGaap, 'StockholdersEquity'),
cash: this.getLatestValue(usGaap, 'CashAndCashEquivalentsAtCarryingValue'),
propertyPlantEquipment: this.getLatestValue(usGaap, 'PropertyPlantAndEquipmentNet'),
// Metadata (from latest available data point)
fiscalYear: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fy,
fiscalQuarter: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.fp,
filingDate: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.filed
? new Date(this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')!.filed)
: undefined,
form: this.getLatestMetadata(usGaap, 'EarningsPerShareDiluted')?.form
};
return fundamentals;
}
/**
* Get the latest value for a US GAAP metric
*/
private getLatestValue(usGaap: any, metricName: string): number | undefined {
const metric = usGaap[metricName];
if (!metric?.units) {
return undefined;
}
// Get the first unit type (USD, shares, etc.)
const unitType = Object.keys(metric.units)[0];
const values = metric.units[unitType];
if (!values || !Array.isArray(values) || values.length === 0) {
return undefined;
}
// Get the latest value (last in array)
const latest = values[values.length - 1];
return latest?.val;
}
/**
* Get metadata from the latest data point
*/
private getLatestMetadata(usGaap: any, metricName: string): any | undefined {
const metric = usGaap[metricName];
if (!metric?.units) {
return undefined;
}
const unitType = Object.keys(metric.units)[0];
const values = metric.units[unitType];
if (!values || !Array.isArray(values) || values.length === 0) {
return undefined;
}
return values[values.length - 1];
}
/**
* Check if SEC EDGAR API is available
* Uses native fetch for automatic gzip decompression
*/
public async isAvailable(): Promise<boolean> {
try {
// Test with Apple's well-known CIK
const url = `${this.baseUrl}/companyfacts/CIK0000320193.json`;
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 5000);
const response = await fetch(url, {
headers: {
'User-Agent': this.userAgent,
'Accept': 'application/json'
},
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok) {
return false;
}
const data = await response.json();
return data && data.facts !== undefined;
} catch (error) {
this.logger.warn('SEC EDGAR provider is not available:', error);
return false;
}
}
/**
* Get cache statistics
*/
public getCacheStats(): {
cikCacheSize: number;
hasTickerList: boolean;
} {
return {
cikCacheSize: this.cikCache.size,
hasTickerList: this.tickerListCache !== null
};
}
/**
* Clear all caches
*/
public clearCache(): void {
this.cikCache.clear();
this.tickerListCache = null;
this.logger.log('SEC EDGAR cache cleared');
}
}

View File

@@ -0,0 +1,194 @@
import * as plugins from '../../plugins.js';
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
import type {
IStockPrice,
IStockDataRequest,
IStockCurrentRequest,
IStockBatchCurrentRequest
} from '../interfaces/stockprice.js';
export class YahooFinanceProvider implements IStockProvider {
public name = 'Yahoo Finance';
public priority = 100;
public readonly requiresAuth = false;
public readonly rateLimit = {
requestsPerMinute: 100, // Conservative estimate
requestsPerDay: undefined
};
private logger = console;
private baseUrl = 'https://query1.finance.yahoo.com';
private userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
constructor(private config?: IProviderConfig) {}
/**
* Unified data fetching method
*/
public async fetchData(request: IStockDataRequest): Promise<IStockPrice | IStockPrice[]> {
switch (request.type) {
case 'current':
return this.fetchCurrentPrice(request);
case 'batch':
return this.fetchBatchCurrentPrices(request);
case 'historical':
throw new Error('Yahoo Finance provider does not support historical data. Use Marketstack provider instead.');
case 'intraday':
throw new Error('Yahoo Finance provider does not support intraday data yet. Use Marketstack provider instead.');
default:
throw new Error(`Unsupported request type: ${(request as any).type}`);
}
}
/**
* Fetch current price for a single ticker
*/
private async fetchCurrentPrice(request: IStockCurrentRequest): Promise<IStockPrice> {
try {
const url = `${this.baseUrl}/v8/finance/chart/${request.ticker}`;
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers({
'User-Agent': this.userAgent
})
.timeout(this.config?.timeout || 10000)
.get();
const responseData = await response.json() as any;
if (!responseData?.chart?.result?.[0]) {
throw new Error(`No data found for ticker ${request.ticker}`);
}
const data = responseData.chart.result[0];
const meta = data.meta;
if (!meta.regularMarketPrice) {
throw new Error(`No price data available for ${request.ticker}`);
}
const stockPrice: IStockPrice = {
ticker: request.ticker.toUpperCase(),
price: meta.regularMarketPrice,
currency: meta.currency || 'USD',
change: meta.regularMarketPrice - meta.previousClose,
changePercent: ((meta.regularMarketPrice - meta.previousClose) / meta.previousClose) * 100,
previousClose: meta.previousClose,
timestamp: new Date(meta.regularMarketTime * 1000),
provider: this.name,
marketState: this.determineMarketState(meta),
exchange: meta.exchange,
exchangeName: meta.exchangeName,
dataType: 'live', // Yahoo provides real-time/near real-time data
fetchedAt: new Date()
};
return stockPrice;
} catch (error) {
console.error(`Failed to fetch price for ${request.ticker}:`, error);
throw new Error(`Yahoo Finance: Failed to fetch price for ${request.ticker}: ${error.message}`);
}
}
/**
* Fetch batch current prices
*/
private async fetchBatchCurrentPrices(request: IStockBatchCurrentRequest): Promise<IStockPrice[]> {
try {
const symbols = request.tickers.join(',');
const url = `${this.baseUrl}/v8/finance/spark?symbols=${symbols}&range=1d&interval=5m`;
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.headers({
'User-Agent': this.userAgent
})
.timeout(this.config?.timeout || 15000)
.get();
const responseData = await response.json() as any;
const prices: IStockPrice[] = [];
for (const [ticker, data] of Object.entries(responseData)) {
if (!data || typeof data !== 'object') continue;
const sparkData = data as any;
if (!sparkData.previousClose || !sparkData.close?.length) {
console.warn(`Incomplete data for ${ticker}, skipping`);
continue;
}
const currentPrice = sparkData.close[sparkData.close.length - 1];
const timestamp = sparkData.timestamp?.[sparkData.timestamp.length - 1];
prices.push({
ticker: ticker.toUpperCase(),
price: currentPrice,
currency: sparkData.currency || 'USD',
change: currentPrice - sparkData.previousClose,
changePercent: ((currentPrice - sparkData.previousClose) / sparkData.previousClose) * 100,
previousClose: sparkData.previousClose,
timestamp: timestamp ? new Date(timestamp * 1000) : new Date(),
provider: this.name,
marketState: sparkData.marketState || 'REGULAR',
exchange: sparkData.exchange,
exchangeName: sparkData.exchangeName,
dataType: 'live', // Yahoo provides real-time/near real-time data
fetchedAt: new Date()
});
}
if (prices.length === 0) {
throw new Error('No valid price data received from batch request');
}
return prices;
} catch (error) {
console.error(`Failed to fetch batch prices:`, error);
throw new Error(`Yahoo Finance: Failed to fetch batch prices: ${error.message}`);
}
}
public async isAvailable(): Promise<boolean> {
try {
// Test with a well-known ticker
await this.fetchData({ type: 'current', ticker: 'AAPL' });
return true;
} catch (error) {
console.warn('Yahoo Finance provider is not available:', error);
return false;
}
}
public supportsMarket(market: string): boolean {
// Yahoo Finance supports most major markets
const supportedMarkets = ['US', 'UK', 'DE', 'FR', 'JP', 'CN', 'HK', 'AU', 'CA'];
return supportedMarkets.includes(market.toUpperCase());
}
public supportsTicker(ticker: string): boolean {
// Basic validation - Yahoo supports most tickers
return /^[A-Z0-9\.\-]{1,10}$/.test(ticker.toUpperCase());
}
private determineMarketState(meta: any): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' {
const marketState = meta.marketState?.toUpperCase();
switch (marketState) {
case 'PRE':
return 'PRE';
case 'POST':
return 'POST';
case 'REGULAR':
return 'REGULAR';
default:
// Check if market is currently open based on timestamps
const now = Date.now() / 1000;
const regularMarketTime = meta.regularMarketTime;
const timeDiff = now - regularMarketTime;
// If last update was more than 1 hour ago, market is likely closed
return timeDiff > 3600 ? 'CLOSED' : 'REGULAR';
}
}
}