Compare commits
37 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1b1324d0f9 | |||
| 71a5a32198 | |||
| 3dbf194320 | |||
| a29a50e825 | |||
| daeff1ce93 | |||
| 298172c00b | |||
| df677b38fb | |||
| c344e47ae6 | |||
| 209af50a4c | |||
| 1d0d44dc29 | |||
| c4d6403721 | |||
| 84cab94beb | |||
| d17683cd67 | |||
| 39537c0568 | |||
| b768b67641 | |||
| 25147deb7f | |||
| 4030bef7a8 | |||
| c6964f0310 | |||
| 9a9f203af2 | |||
| 174086defc | |||
| 43c9d3b3b6 | |||
| 39724b61d6 | |||
| d9588f8f65 | |||
| 6ce6153ccf | |||
| ec2d4f9fbc | |||
| a19be31381 | |||
| 9c3f012da7 | |||
| 8ebbc16bcd | |||
| c177193438 | |||
| 7c07bc59e4 | |||
| e4a8d371f7 | |||
| 1c0e04cb0d | |||
| c3f6ef531b | |||
| a67a0993d6 | |||
| bc43e4c44a | |||
| 9b2dcd7377 | |||
| 1eda50ad13 |
108
changelog.md
108
changelog.md
@@ -1,5 +1,113 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-07-12 - 1.6.0 - feat(readme)
|
||||
Revamp documentation and package description for enhanced clarity
|
||||
|
||||
- Restructured README to highlight real-time stock data and German business data, streamlining quick start and advanced examples
|
||||
- Updated package.json description to better reflect library capabilities
|
||||
- Added .claude/settings.local.json to define permissions for external tools
|
||||
- Refined code examples in tests and documentation for improved clarity and consistency
|
||||
|
||||
## 2025-04-09 - 1.5.3 - fix(test)
|
||||
Await file writes in Handelsregister tests to ensure all downloads complete before test end
|
||||
|
||||
- Replaced array.map with await Promise.all to properly await asynchronous file writes in test/test.handelsregister.ts
|
||||
- Improved robustness of asynchronous operations in test suite
|
||||
|
||||
## 2025-04-09 - 1.5.2 - fix(readme)
|
||||
Improve .env configuration code block formatting in documentation
|
||||
|
||||
- Wrap the .env variables block in triple backticks for clarity
|
||||
- Ensure consistency in the Markdown styling of code snippets
|
||||
|
||||
## 2025-04-09 - 1.5.1 - fix(core)
|
||||
No changes detected in project files or documentation. This commit is a placeholder to record that nothing was updated.
|
||||
|
||||
|
||||
## 2025-04-09 - 1.5.0 - feat(documentation)
|
||||
Enhance project metadata and documentation with comprehensive usage examples, updated descriptions, and improved keywords.
|
||||
|
||||
- Updated npmextra.json and package.json to refine the project description and keyword list.
|
||||
- Expanded readme.md with detailed sections on environment setup, CRUD operations, bulk JSONL processing, and advanced Handelsregister integrations.
|
||||
- Included advanced workflow examples and error handling strategies in the documentation.
|
||||
- Adjusted test cases (e.g. in test/test.handelsregister.ts) to reflect changes in company name usage.
|
||||
|
||||
## 2025-04-08 - 1.4.6 - fix(tests & jsonl)
|
||||
Improve test structure and refine JSONL parsing for incomplete data
|
||||
|
||||
- Refactored test files to remove redundant get-specific-company tests in test.ts and added missing tests in test.handelsregister.ts
|
||||
- Updated JSONL data processor to conditionally parse remaining data when available
|
||||
|
||||
## 2025-04-05 - 1.4.5 - fix(metadata)
|
||||
Update repository, bugs, and homepage URLs to code.foss.global
|
||||
|
||||
- Repository URL updated from gitlab.com to code.foss.global
|
||||
- Bugs URL updated from gitlab.com to code.foss.global
|
||||
- Homepage URL updated to code.foss.global
|
||||
|
||||
## 2025-04-05 - 1.4.4 - fix(dependencies & tests)
|
||||
Update dependency versions and refine test search query
|
||||
|
||||
- Bumped versions for several dependencies in package.json, including @git.zone/tsbuild, @git.zone/tsbundle, @git.zone/tstest, @push.rocks/tapbundle, @push.rocks/smartdata, @push.rocks/smartfile, @push.rocks/smartpromise, @push.rocks/smartrequest, and @tsclass/tsclass
|
||||
- Updated test file to replace the search query 'Volkswagen' with 'LADR'
|
||||
- Re-enabled the build initial data test by removing tap.skip
|
||||
|
||||
## 2025-01-07 - 1.4.3 - fix(test)
|
||||
Corrected index value in test for fetching specific company data
|
||||
|
||||
- Updated the index from 8 to 7 for the germanParsedRegistration fetch in test
|
||||
|
||||
## 2025-01-07 - 1.4.2 - fix(core)
|
||||
Fix concurrency and download handling in HandelsRegister class and adjust test cases
|
||||
|
||||
- Improved the clickFindButton function to include an argument for results limit.
|
||||
- Enhanced the downloadFile function to rename and ensure files are correctly handled.
|
||||
- Updated searchCompany method to allow specifying a limit on the number of search results.
|
||||
- Adjusted test cases to select specific test data indices and output test files to a dedicated directory.
|
||||
|
||||
## 2025-01-04 - 1.4.1 - fix(core)
|
||||
Fix issues with JSONL data processing and improve error handling in business record validation
|
||||
|
||||
- Fixed JSONL data processing by adding concurrent processing for each JSON line to enhance performance.
|
||||
- Added validation logic in BusinessRecord class to ensure that the mandatory fields are checked.
|
||||
- Adjusted environment variable loading in OpenData class to ensure correct database initialization.
|
||||
- Included missing dependencies and exports in the project files to ensure proper functionality.
|
||||
|
||||
## 2025-01-04 - 1.4.0 - feat(HandelsRegister)
|
||||
Add file download functionality to HandelsRegister
|
||||
|
||||
- Implemented file download feature in the HandelsRegister class.
|
||||
- Configured pages in Puppeteer to allow downloads and set download paths.
|
||||
- Parsed German registration information with more robust error handling.
|
||||
- Added specific methods for downloading and handling 'SI' and 'AD' files.
|
||||
|
||||
## 2025-01-03 - 1.3.1 - fix(HandelsRegister)
|
||||
Refined HandelsRegister functionality for better error handling and response capture.
|
||||
|
||||
- Improved parsing logic in parseGermanRegistration function.
|
||||
- Enhanced navigateToPage and clickFindButton methods with error messages for clarity.
|
||||
- Implemented a new responseListener to handle and log HTTP responses correctly.
|
||||
|
||||
## 2025-01-03 - 1.3.0 - feat(core)
|
||||
Enhanced data handling capabilities and improved company search functionalities.
|
||||
|
||||
- Updated business record handling to support more registration types.
|
||||
- Improved search capabilities for fetching company data with refined registration type matching.
|
||||
- Added robust logging for JSONL data processing with early exit on successful parse.
|
||||
- Reorganized test cases to include specific company data retrieval.
|
||||
|
||||
## 2025-01-02 - 1.2.1 - fix(BusinessRecord)
|
||||
Add missing field registrationType to BusinessRecord data
|
||||
|
||||
- Introduced the 'registrationType' field to the BusinessRecord data schema with possible values 'HRA' or 'HRB'.
|
||||
|
||||
## 2025-01-02 - 1.2.0 - feat(core)
|
||||
Integrate Handelsregister search for company data retrieval
|
||||
|
||||
- Added support for searching company data via Handelsregister.
|
||||
- Replaced GermanBusinessData functionality with JsonlDataProcessor.
|
||||
- Included smartbrowser dependency for handling web requests to Handelsregister.
|
||||
|
||||
## 2025-01-01 - 1.1.5 - fix(GermanBusinessData)
|
||||
Add console log for total records processed at the end of the stream.
|
||||
|
||||
|
||||
@@ -5,30 +5,34 @@
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "fin.cx",
|
||||
"gitrepo": "opendata",
|
||||
"description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
|
||||
"description": "A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.",
|
||||
"npmPackagename": "@fin.cx/opendata",
|
||||
"license": "MIT",
|
||||
"projectDomain": "fin.cx",
|
||||
"keywords": [
|
||||
"TypeScript",
|
||||
"open data",
|
||||
"business data",
|
||||
"German companies",
|
||||
"data management",
|
||||
"business registry",
|
||||
"npm package",
|
||||
"business data",
|
||||
"MongoDB",
|
||||
"JSONL",
|
||||
"bulk processing",
|
||||
"data management",
|
||||
"automation",
|
||||
"data integration",
|
||||
"database",
|
||||
"data processing",
|
||||
"data retrieval",
|
||||
"data update"
|
||||
"browser automation",
|
||||
"Handelsregister",
|
||||
"web scraping",
|
||||
"file processing",
|
||||
"business registry",
|
||||
"data retrieval"
|
||||
]
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
}
|
||||
}
|
||||
60
package.json
60
package.json
@@ -1,45 +1,51 @@
|
||||
{
|
||||
"name": "@fin.cx/opendata",
|
||||
"version": "1.1.5",
|
||||
"version": "1.6.0",
|
||||
"private": false,
|
||||
"description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
|
||||
"description": "A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"author": "Task Venture Capital GmbH",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --web)",
|
||||
"test": "(tstest test/ --verbose)",
|
||||
"build": "(tsbuild --web --allowimplicitany)",
|
||||
"buildDocs": "(tsdoc)"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.2.0",
|
||||
"@git.zone/tsbundle": "^2.1.0",
|
||||
"@git.zone/tsbuild": "^2.6.4",
|
||||
"@git.zone/tsbundle": "^2.5.1",
|
||||
"@git.zone/tsrun": "^1.3.3",
|
||||
"@git.zone/tstest": "^1.0.90",
|
||||
"@push.rocks/tapbundle": "^5.5.4",
|
||||
"@types/node": "^22.10.2"
|
||||
"@git.zone/tstest": "^2.3.1",
|
||||
"@types/node": "^22.14.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/lik": "^6.2.2",
|
||||
"@push.rocks/qenv": "^6.1.0",
|
||||
"@push.rocks/smartarchive": "^4.0.39",
|
||||
"@push.rocks/smartdata": "^5.2.10",
|
||||
"@push.rocks/smartarray": "^1.1.0",
|
||||
"@push.rocks/smartbrowser": "^2.0.8",
|
||||
"@push.rocks/smartdata": "^5.15.1",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.0.23",
|
||||
"@push.rocks/smartfile": "^11.2.5",
|
||||
"@push.rocks/smartlog": "^3.1.8",
|
||||
"@push.rocks/smartpath": "^5.0.18",
|
||||
"@push.rocks/smartpromise": "^4.0.4",
|
||||
"@push.rocks/smartrequest": "^2.0.23",
|
||||
"@push.rocks/smartstream": "^3.2.5"
|
||||
"@push.rocks/smartpromise": "^4.2.3",
|
||||
"@push.rocks/smartrequest": "^2.1.0",
|
||||
"@push.rocks/smartstream": "^3.2.5",
|
||||
"@push.rocks/smartunique": "^3.0.9",
|
||||
"@push.rocks/smartxml": "^1.1.1",
|
||||
"@tsclass/tsclass": "^9.2.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://gitlab.com/fin.cx/opendata.git"
|
||||
"url": "https://code.foss.global/fin.cx/opendata.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://gitlab.com/fin.cx/opendata/issues"
|
||||
"url": "https://code.foss.global/fin.cx/opendata/issues"
|
||||
},
|
||||
"homepage": "https://gitlab.com/fin.cx/opendata#readme",
|
||||
"homepage": "https://code.foss.global/fin.cx/opendata#readme",
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
],
|
||||
@@ -58,17 +64,19 @@
|
||||
"keywords": [
|
||||
"TypeScript",
|
||||
"open data",
|
||||
"business data",
|
||||
"German companies",
|
||||
"data management",
|
||||
"business registry",
|
||||
"npm package",
|
||||
"business data",
|
||||
"MongoDB",
|
||||
"JSONL",
|
||||
"bulk processing",
|
||||
"data management",
|
||||
"automation",
|
||||
"data integration",
|
||||
"database",
|
||||
"data processing",
|
||||
"data retrieval",
|
||||
"data update"
|
||||
]
|
||||
"browser automation",
|
||||
"Handelsregister",
|
||||
"web scraping",
|
||||
"file processing",
|
||||
"business registry",
|
||||
"data retrieval"
|
||||
],
|
||||
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
|
||||
}
|
||||
|
||||
4878
pnpm-lock.yaml
generated
4878
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
# OpenData Project Hints
|
||||
|
||||
## Stocks Module
|
||||
|
||||
### Overview
|
||||
The stocks module provides real-time stock price data through various provider implementations. Currently supports Yahoo Finance with an extensible architecture for additional providers.
|
||||
|
||||
### Architecture
|
||||
- **Provider Pattern**: Each stock data source implements the `IStockProvider` interface
|
||||
- **Service Registry**: `StockPriceService` manages providers with priority-based selection
|
||||
- **Caching**: Built-in cache with configurable TTL to reduce API calls
|
||||
- **Fallback Logic**: Automatic failover between providers if one fails
|
||||
|
||||
### Yahoo Finance Provider Notes
|
||||
- Uses public API endpoints (no authentication required)
|
||||
- Two main endpoints:
|
||||
- `/v8/finance/chart/{ticker}` - Single ticker with full data
|
||||
- `/v8/finance/spark?symbols={tickers}` - Multiple tickers with basic data
|
||||
- Response data is in `response.body` when using smartrequest
|
||||
- Requires User-Agent header to avoid rate limiting
|
||||
|
||||
### Usage Example
|
||||
```typescript
|
||||
import { StockPriceService, YahooFinanceProvider } from '@fin.cx/opendata';
|
||||
|
||||
const stockService = new StockPriceService({ ttl: 60000 });
|
||||
const yahooProvider = new YahooFinanceProvider();
|
||||
stockService.register(yahooProvider);
|
||||
|
||||
const price = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
console.log(`${price.ticker}: $${price.price}`);
|
||||
```
|
||||
|
||||
### Testing
|
||||
- Tests use real API calls (be mindful of rate limits)
|
||||
- Mock invalid ticker 'INVALID_TICKER_XYZ' for error testing
|
||||
- Clear cache between tests to ensure fresh data
|
||||
- The spark endpoint may return fewer results than requested
|
||||
|
||||
### Future Providers
|
||||
To add a new provider:
|
||||
1. Create `ts/stocks/providers/provider.{name}.ts`
|
||||
2. Implement the `IStockProvider` interface
|
||||
3. Register with `StockPriceService`
|
||||
4. No changes needed to existing code
|
||||
456
readme.md
456
readme.md
@@ -1,218 +1,314 @@
|
||||
# @fin.cx/opendata
|
||||
🚀 **Real-time financial data and German business intelligence in one powerful TypeScript library**
|
||||
|
||||
A TypeScript-based library for accessing and managing open business data, specifically for German companies.
|
||||
Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API.
|
||||
|
||||
## Install
|
||||
|
||||
To install the `@fin.cx/opendata` package, you can use npm or yarn as your package manager. Here's how you can do it:
|
||||
|
||||
Using npm:
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @fin.cx/opendata
|
||||
```
|
||||
|
||||
Using yarn:
|
||||
|
||||
```bash
|
||||
# or
|
||||
yarn add @fin.cx/opendata
|
||||
```
|
||||
|
||||
## Usage
|
||||
## Quick Start
|
||||
|
||||
The `@fin.cx/opendata` package provides a comprehensive set of functionalities for handling open business data, focusing on German business data. Let's explore the detailed capabilities of this library through extensive examples and instructions.
|
||||
### 📈 Real-Time Stock Data
|
||||
|
||||
### Setting Up the Environment
|
||||
Get live market data in seconds:
|
||||
|
||||
First, make sure you've set up the necessary environment variables for MongoDB. You will need the following environment variables:
|
||||
- `MONGODB_URL`: The URL for your MongoDB instance.
|
||||
- `MONGODB_NAME`: The name of the database to use.
|
||||
- `MONGODB_USER`: A valid username for accessing the database.
|
||||
- `MONGODB_PASS`: The password associated with the MongoDB user.
|
||||
```typescript
|
||||
import { StockPriceService, YahooFinanceProvider } from '@fin.cx/opendata';
|
||||
|
||||
These variables can be configured in a `.env` file or managed through a specific service used for secure environment variables handling.
|
||||
// Initialize the service
|
||||
const stockService = new StockPriceService();
|
||||
stockService.register(new YahooFinanceProvider());
|
||||
|
||||
### Importing and Initializing the Library
|
||||
// Get single stock price
|
||||
const apple = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
console.log(`Apple: $${apple.price} (${apple.changePercent.toFixed(2)}%)`);
|
||||
|
||||
To start working with the library, import the necessary classes and initialize the `OpenData` class.
|
||||
// Get multiple prices at once
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: ['AAPL', 'MSFT', 'GOOGL', 'BTC-USD', 'ETH-USD']
|
||||
});
|
||||
|
||||
// Market indices, crypto, forex, commodities - all supported!
|
||||
const marketData = await stockService.getPrices({
|
||||
tickers: ['^GSPC', '^DJI', 'BTC-USD', 'EURUSD=X', 'GC=F']
|
||||
});
|
||||
```
|
||||
|
||||
### 🏢 German Business Data
|
||||
|
||||
Access comprehensive data on German companies:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
const initializeOpenData = async () => {
|
||||
const openData = new OpenData();
|
||||
const openData = new OpenData();
|
||||
await openData.start();
|
||||
|
||||
// Create a business record
|
||||
const company = new openData.CBusinessRecord();
|
||||
company.data = {
|
||||
name: "TechStart GmbH",
|
||||
city: "Berlin",
|
||||
registrationId: "HRB 123456",
|
||||
// ... more fields
|
||||
};
|
||||
await company.save();
|
||||
|
||||
// Search companies by city
|
||||
const berlinCompanies = await openData.db
|
||||
.collection('businessrecords')
|
||||
.find({ city: "Berlin" })
|
||||
.toArray();
|
||||
|
||||
// Import bulk data from official sources
|
||||
await openData.buildInitialDb();
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### 🎯 Stock Market Module
|
||||
|
||||
- **Real-time prices** for stocks, ETFs, indices, crypto, forex, and commodities
|
||||
- **Batch operations** - fetch 100+ symbols in one request
|
||||
- **Smart caching** - configurable TTL, automatic invalidation
|
||||
- **Provider system** - easily extensible for new data sources
|
||||
- **Automatic retries** and fallback mechanisms
|
||||
- **Type-safe** - full TypeScript support with detailed interfaces
|
||||
|
||||
### 🇩🇪 German Business Intelligence
|
||||
|
||||
- **MongoDB integration** for scalable data storage
|
||||
- **Bulk JSONL import** from official German data sources
|
||||
- **Handelsregister automation** - automated document retrieval
|
||||
- **CRUD operations** with validation
|
||||
- **Streaming processing** for multi-GB datasets
|
||||
|
||||
## Advanced Examples
|
||||
|
||||
### Market Dashboard
|
||||
|
||||
Create a real-time market overview:
|
||||
|
||||
```typescript
|
||||
const indicators = [
|
||||
// Indices
|
||||
{ ticker: '^GSPC', name: 'S&P 500' },
|
||||
{ ticker: '^IXIC', name: 'NASDAQ' },
|
||||
|
||||
// Tech Giants
|
||||
{ ticker: 'AAPL', name: 'Apple' },
|
||||
{ ticker: 'MSFT', name: 'Microsoft' },
|
||||
|
||||
// Crypto
|
||||
{ ticker: 'BTC-USD', name: 'Bitcoin' },
|
||||
{ ticker: 'ETH-USD', name: 'Ethereum' },
|
||||
|
||||
// Commodities
|
||||
{ ticker: 'GC=F', name: 'Gold' },
|
||||
{ ticker: 'CL=F', name: 'Oil' }
|
||||
];
|
||||
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: indicators.map(i => i.ticker)
|
||||
});
|
||||
|
||||
// Display with color-coded changes
|
||||
prices.forEach(price => {
|
||||
const indicator = indicators.find(i => i.ticker === price.ticker);
|
||||
const arrow = price.change >= 0 ? '↑' : '↓';
|
||||
const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
|
||||
|
||||
console.log(
|
||||
`${indicator.name.padEnd(15)} ${price.price.toFixed(2).padStart(10)} ` +
|
||||
`${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m`
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
### Handelsregister Integration
|
||||
|
||||
Automate German company data retrieval:
|
||||
|
||||
```typescript
|
||||
// Search for a company
|
||||
const results = await openData.handelsregister.searchCompany("Siemens AG");
|
||||
|
||||
// Get detailed information and documents
|
||||
const details = await openData.handelsregister.getSpecificCompany({
|
||||
court: "Munich",
|
||||
type: "HRB",
|
||||
number: "6684"
|
||||
});
|
||||
|
||||
// Downloaded files include:
|
||||
// - XML data (SI files)
|
||||
// - PDF documents (AD files)
|
||||
for (const file of details.files) {
|
||||
await file.writeToDir('./downloads');
|
||||
}
|
||||
```
|
||||
|
||||
### Combined Data Analysis
|
||||
|
||||
Merge financial and business data:
|
||||
|
||||
```typescript
|
||||
// Find all public German companies (AG)
|
||||
const publicCompanies = await openData.db
|
||||
.collection('businessrecords')
|
||||
.find({ legalForm: 'AG' })
|
||||
.toArray();
|
||||
|
||||
// Enrich with stock data
|
||||
for (const company of publicCompanies) {
|
||||
try {
|
||||
await openData.start();
|
||||
console.log('OpenData instance has started successfully.');
|
||||
// Map company to ticker (custom logic needed)
|
||||
const ticker = mapCompanyToTicker(company.data.name);
|
||||
|
||||
// Example usage:
|
||||
await createAndManageBusinessRecords(openData);
|
||||
if (ticker) {
|
||||
const stock = await stockService.getPrice({ ticker });
|
||||
|
||||
// Add financial metrics
|
||||
company.data.stockPrice = stock.price;
|
||||
company.data.marketCap = stock.price * getSharesOutstanding(ticker);
|
||||
company.data.priceChange = stock.changePercent;
|
||||
|
||||
await company.save();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error starting OpenData:', error);
|
||||
} finally {
|
||||
await openData.stop();
|
||||
console.log('OpenData instance has stopped.');
|
||||
}
|
||||
};
|
||||
|
||||
initializeOpenData();
|
||||
```
|
||||
|
||||
### Managing Business Records
|
||||
|
||||
The `BusinessRecord` class represents a company's data. Let's explore how you can create, retrieve, update, and manage these records.
|
||||
|
||||
#### Creating a New BusinessRecord
|
||||
|
||||
Creating a new business record involves instantiating the `BusinessRecord` class and setting the relevant properties.
|
||||
|
||||
```typescript
|
||||
import { BusinessRecord } from '@fin.cx/opendata';
|
||||
|
||||
const createBusinessRecord = async (openData: OpenData) => {
|
||||
const businessRecord = new openData.CBusinessRecord();
|
||||
|
||||
businessRecord.data = {
|
||||
name: "Tech Innovations GmbH",
|
||||
address: "Tech Park 42",
|
||||
postalCode: "80333",
|
||||
city: "Munich",
|
||||
country: "Germany",
|
||||
phone: "+49 89 123456",
|
||||
email: "info@techinnovations.de",
|
||||
website: "https://techinnovations.de",
|
||||
businessType: "GmbH",
|
||||
registrationNumber: "HRB 654321",
|
||||
registrationCourt: "Munich",
|
||||
legalForm: "GmbH",
|
||||
managingDirectors: ["Alice Müller", "Bob Schmidt"],
|
||||
foundingDate: new Date("2020-01-01").toISOString(),
|
||||
capital: "100,000 EUR",
|
||||
purpose: "Developing innovative tech solutions",
|
||||
lastUpdate: new Date().toISOString()
|
||||
};
|
||||
|
||||
await businessRecord.save();
|
||||
console.log('BusinessRecord saved:', businessRecord);
|
||||
};
|
||||
```
|
||||
|
||||
#### Retrieving Business Records
|
||||
|
||||
You can retrieve existing business records by querying the database using various data fields.
|
||||
|
||||
```typescript
|
||||
const retrieveBusinessRecords = async (openData: OpenData) => {
|
||||
const records = await openData.db
|
||||
.collection<BusinessRecord>('businessrecords')
|
||||
.find({ city: "Munich" })
|
||||
.toArray();
|
||||
|
||||
console.log('Found Business Records:', records);
|
||||
};
|
||||
```
|
||||
|
||||
#### Updating Existing Records
|
||||
|
||||
To update an existing `BusinessRecord`, you retrieve the record, modify its data, and save it again.
|
||||
|
||||
```typescript
|
||||
const updateBusinessRecord = async (openData: OpenData, recordId: string) => {
|
||||
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
|
||||
if (businessRecord) {
|
||||
businessRecord.data.phone = "+49 89 987654";
|
||||
businessRecord.data.lastUpdate = new Date().toISOString();
|
||||
|
||||
await businessRecord.save();
|
||||
console.log('BusinessRecord updated:', businessRecord);
|
||||
} else {
|
||||
console.log('BusinessRecord not found for id:', recordId);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
#### Deleting a Business Record
|
||||
|
||||
You can delete a business record using its unique identifier.
|
||||
|
||||
```typescript
|
||||
const deleteBusinessRecord = async (openData: OpenData, recordId: string) => {
|
||||
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
|
||||
if (businessRecord) {
|
||||
await businessRecord.delete();
|
||||
console.log(`BusinessRecord with id ${recordId} deleted successfully.`);
|
||||
} else {
|
||||
console.log('No record found for the provided id:', recordId);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
### Updating German Business Data
|
||||
|
||||
The package includes functionalities to keep your business data up-to-date by downloading from official German open data repositories.
|
||||
|
||||
```typescript
|
||||
const updateGermanBusinessData = async (openData: OpenData) => {
|
||||
try {
|
||||
await openData.germanBusinesses.update();
|
||||
console.log('German business data has been updated successfully.');
|
||||
} catch (error) {
|
||||
console.error('Error updating German business data:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
This function fetches the latest open data regarding German companies, processes it, and updates your local database.
|
||||
|
||||
### Detailed Class Insights
|
||||
|
||||
#### OpenData Class
|
||||
|
||||
The `OpenData` class serves as the core of the library, initializing necessary components and controlling data flows:
|
||||
|
||||
- **db**: Represents the connection to your MongoDB database.
|
||||
- **germanBusinesses**: An instance handling specific operations related to German business data updates.
|
||||
|
||||
```typescript
|
||||
class OpenData {
|
||||
db: plugins.smartdata.SmartdataDb;
|
||||
germanBusinesses: GermanBusinessData;
|
||||
|
||||
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
|
||||
|
||||
public async start() {
|
||||
// Database initialization logic
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
// Cleanup logic
|
||||
// Handle missing tickers gracefully
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### GermanBusinessData Class
|
||||
## Configuration
|
||||
|
||||
This class deals specifically with German company data — fetching, processing, and updating local databases consistently with official German data sources.
|
||||
### Stock Service Options
|
||||
|
||||
```typescript
|
||||
class GermanBusinessData {
|
||||
public async start() {
|
||||
await this.update();
|
||||
}
|
||||
const stockService = new StockPriceService({
|
||||
ttl: 60000, // Cache for 1 minute
|
||||
maxEntries: 1000 // Max cached symbols
|
||||
});
|
||||
|
||||
public async update() {
|
||||
// Logic for updating business data using import streams and parsing JSON lines.
|
||||
}
|
||||
// Provider configuration
|
||||
stockService.register(new YahooFinanceProvider(), {
|
||||
enabled: true,
|
||||
priority: 100,
|
||||
timeout: 10000,
|
||||
retryAttempts: 3,
|
||||
retryDelay: 1000
|
||||
});
|
||||
```
|
||||
|
||||
### MongoDB Setup
|
||||
|
||||
Set environment variables:
|
||||
|
||||
```env
|
||||
MONGODB_URL=mongodb://localhost:27017
|
||||
MONGODB_NAME=opendata
|
||||
MONGODB_USER=myuser
|
||||
MONGODB_PASS=mypass
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Stock Types
|
||||
|
||||
```typescript
|
||||
interface IStockPrice {
|
||||
ticker: string;
|
||||
price: number;
|
||||
currency: string;
|
||||
change: number;
|
||||
changePercent: number;
|
||||
previousClose: number;
|
||||
timestamp: Date;
|
||||
provider: string;
|
||||
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
|
||||
exchange?: string;
|
||||
exchangeName?: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Ensuring Data Accuracy and Integrity
|
||||
### Key Methods
|
||||
|
||||
When working with business data, ensuring integrity and accuracy is crucial. Each record should be checked for validity before being saved or updated, minimizing inconsistencies. Moreover, robust error handling is essential in every step, from data retrieval to database operations, particularly when dealing with external data sources.
|
||||
**StockPriceService**
|
||||
- `getPrice(request)` - Single stock price
|
||||
- `getPrices(request)` - Batch prices
|
||||
- `register(provider)` - Add data provider
|
||||
- `clearCache()` - Clear price cache
|
||||
|
||||
The `@fin.cx/opendata` module provides an extensive toolset for accessing and managing business data, particularly for companies based in Germany. Its functionalities include creating, updating, retrieving, and deleting business records, as well as keeping them current with the latest open data releases. This makes it an invaluable asset for developers aiming to integrate open data seamlessly into their systems, ensuring robust data management capabilities within their applications.
|
||||
**OpenData**
|
||||
- `start()` - Initialize MongoDB connection
|
||||
- `buildInitialDb()` - Import bulk data
|
||||
- `CBusinessRecord` - Business record class
|
||||
- `handelsregister` - Registry automation
|
||||
|
||||
Happy exploring and integrating open data into your projects!
|
||||
undefined
|
||||
## Performance
|
||||
|
||||
- **Batch fetching**: Get 100+ prices in <500ms
|
||||
- **Caching**: Instant repeated queries
|
||||
- **Concurrent processing**: Handle 1000+ records/second
|
||||
- **Streaming**: Process GB-sized datasets without memory issues
|
||||
|
||||
## Extensibility
|
||||
|
||||
The provider architecture makes it easy to add new data sources:
|
||||
|
||||
```typescript
|
||||
class MyCustomProvider implements IStockProvider {
|
||||
name = 'My Provider';
|
||||
|
||||
async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
// Your implementation
|
||||
}
|
||||
|
||||
// ... other required methods
|
||||
}
|
||||
|
||||
stockService.register(new MyCustomProvider());
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
Run the comprehensive test suite:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
View live market data:
|
||||
|
||||
```bash
|
||||
npm test -- --grep "market indicators"
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
We welcome contributions! Please see our contributing guidelines for details.
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
193
readme.plan.md
Normal file
193
readme.plan.md
Normal file
@@ -0,0 +1,193 @@
|
||||
# Stock Prices Module Implementation Plan
|
||||
|
||||
Command to reread guidelines: Read /home/philkunz/.claude/CLAUDE.md
|
||||
|
||||
## Overview
|
||||
Implementation of a stocks module for fetching current stock prices using various APIs. The architecture will support multiple providers, but we'll start with implementing only Yahoo Finance API. The design will make it easy to add additional providers (Alpha Vantage, IEX Cloud, etc.) in the future without changing the core architecture.
|
||||
|
||||
## Phase 1: Yahoo Finance Implementation
|
||||
|
||||
### 1.1 Research & Documentation
|
||||
- [ ] Research Yahoo Finance API endpoints (no official API, using public endpoints)
|
||||
- [ ] Document available data fields and formats
|
||||
- [ ] Identify rate limits and restrictions
|
||||
- [ ] Test endpoints manually with curl
|
||||
|
||||
### 1.2 Module Structure
|
||||
```
|
||||
ts/
|
||||
├── index.ts # Main exports
|
||||
├── plugins.ts # External dependencies
|
||||
└── stocks/
|
||||
├── index.ts # Stocks module exports
|
||||
├── classes.stockservice.ts # Main StockPriceService class
|
||||
├── interfaces/
|
||||
│ ├── stockprice.ts # IStockPrice interface
|
||||
│ └── provider.ts # IStockProvider interface (for all providers)
|
||||
└── providers/
|
||||
├── provider.yahoo.ts # Yahoo Finance implementation
|
||||
└── (future: provider.alphavantage.ts, provider.iex.ts, etc.)
|
||||
```
|
||||
|
||||
### 1.3 Core Interfaces
|
||||
```typescript
|
||||
// IStockPrice - Standardized stock price data
|
||||
interface IStockPrice {
|
||||
ticker: string;
|
||||
price: number;
|
||||
currency: string;
|
||||
change: number;
|
||||
changePercent: number;
|
||||
timestamp: Date;
|
||||
provider: string;
|
||||
}
|
||||
|
||||
// IStockProvider - Provider implementation contract
|
||||
interface IStockProvider {
|
||||
name: string;
|
||||
fetchPrice(ticker: string): Promise<IStockPrice>;
|
||||
fetchPrices(tickers: string[]): Promise<IStockPrice[]>;
|
||||
isAvailable(): Promise<boolean>;
|
||||
}
|
||||
```
|
||||
|
||||
### 1.4 Yahoo Finance Provider Implementation
|
||||
- [ ] Create YahooFinanceProvider class
|
||||
- [ ] Implement HTTP requests to Yahoo Finance endpoints
|
||||
- [ ] Parse response data into IStockPrice format
|
||||
- [ ] Handle errors and edge cases
|
||||
- [ ] Add request throttling/rate limiting
|
||||
|
||||
### 1.5 Main Service Class
|
||||
- [ ] Create StockPriceService class with provider registry
|
||||
- [ ] Implement provider interface for pluggable providers
|
||||
- [ ] Register Yahoo provider (with ability to add more later)
|
||||
- [ ] Add method for single ticker lookup
|
||||
- [ ] Add method for batch ticker lookup
|
||||
- [ ] Implement error handling with graceful degradation
|
||||
- [ ] Design fallback mechanism (ready for multiple providers)
|
||||
|
||||
## Phase 2: Core Features
|
||||
|
||||
### 2.1 Service Architecture
|
||||
- [ ] Create provider registry pattern for managing multiple providers
|
||||
- [ ] Implement provider priority and selection logic
|
||||
- [ ] Design provider health check interface
|
||||
- [ ] Create provider configuration system
|
||||
- [ ] Implement provider discovery mechanism
|
||||
- [ ] Add provider capability querying (which tickers/markets supported)
|
||||
|
||||
## Phase 3: Advanced Features
|
||||
|
||||
### 3.1 Caching System
|
||||
- [ ] Design cache interface
|
||||
- [ ] Implement in-memory cache with TTL
|
||||
- [ ] Add cache invalidation logic
|
||||
- [ ] Make cache configurable per ticker
|
||||
|
||||
### 3.2 Configuration
|
||||
- [ ] Provider configuration (timeout, retry settings)
|
||||
- [ ] Cache configuration (TTL, max entries)
|
||||
- [ ] Request timeout configuration
|
||||
- [ ] Error handling configuration
|
||||
|
||||
### 3.3 Error Handling
|
||||
- [ ] Define custom error types
|
||||
- [ ] Implement retry logic with exponential backoff
|
||||
- [ ] Add circuit breaker pattern for failing providers
|
||||
- [ ] Comprehensive error logging
|
||||
|
||||
## Phase 4: Testing
|
||||
|
||||
### 4.1 Unit Tests
|
||||
- [ ] Test each provider independently
|
||||
- [ ] Mock HTTP requests for predictable testing
|
||||
- [ ] Test error scenarios
|
||||
- [ ] Test data transformation logic
|
||||
|
||||
### 4.2 Integration Tests
|
||||
- [ ] Test with real API calls (rate limit aware)
|
||||
- [ ] Test provider fallback scenarios
|
||||
- [ ] Test batch operations
|
||||
- [ ] Test cache behavior
|
||||
|
||||
### 4.3 Performance Tests
|
||||
- [ ] Measure response times
|
||||
- [ ] Test concurrent request handling
|
||||
- [ ] Validate cache effectiveness
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. **Week 1: Yahoo Finance Provider**
|
||||
- Research and test Yahoo endpoints
|
||||
- Implement basic provider and service
|
||||
- Create core interfaces
|
||||
- Basic error handling
|
||||
|
||||
2. **Week 2: Service Architecture**
|
||||
- Create extensible provider system
|
||||
- Implement provider interface
|
||||
- Add provider registration
|
||||
|
||||
3. **Week 3: Advanced Features**
|
||||
- Implement caching system
|
||||
- Add configuration management
|
||||
- Enhance error handling
|
||||
|
||||
4. **Week 4: Testing & Documentation**
|
||||
- Write comprehensive tests
|
||||
- Create usage documentation
|
||||
- Performance optimization
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Required
|
||||
- `@push.rocks/smartrequest` - HTTP requests
|
||||
- `@push.rocks/smartpromise` - Promise utilities
|
||||
- `@push.rocks/smartlog` - Logging
|
||||
|
||||
### Development
|
||||
- `@git.zone/tstest` - Testing framework
|
||||
- `@git.zone/tsrun` - TypeScript execution
|
||||
|
||||
## API Endpoints Research
|
||||
|
||||
### Yahoo Finance
|
||||
- Base URL: `https://query1.finance.yahoo.com/v8/finance/chart/{ticker}`
|
||||
- No authentication required
|
||||
- Returns JSON with price data
|
||||
- Rate limits unknown (need to test)
|
||||
- Alternative endpoints to explore:
|
||||
- `/v7/finance/quote` - Simplified quote data
|
||||
- `/v10/finance/quoteSummary` - Detailed company data
|
||||
|
||||
## Success Criteria
|
||||
|
||||
1. Can fetch current stock prices for any valid ticker
|
||||
2. Extensible architecture for future providers
|
||||
3. Response time < 1 second for cached data
|
||||
4. Response time < 3 seconds for fresh data
|
||||
5. Proper error handling and recovery
|
||||
6. Comprehensive test coverage (>80%)
|
||||
|
||||
## Notes
|
||||
|
||||
- Yahoo Finance provides free stock data without authentication
|
||||
- **Architecture designed for multiple providers**: While only implementing Yahoo Finance initially, all interfaces, classes, and patterns are designed to support multiple stock data providers
|
||||
- The provider registry pattern allows adding new providers without modifying existing code
|
||||
- Each provider implements the same IStockProvider interface for consistency
|
||||
- Future providers can be added by simply creating a new provider class and registering it
|
||||
- Implement proper TypeScript types for all data structures
|
||||
- Follow the project's coding standards (prefix interfaces with 'I')
|
||||
- Use plugins.ts for all external dependencies
|
||||
- Keep filenames lowercase
|
||||
- Write tests using @git.zone/tstest with smartexpect syntax
|
||||
- Focus on clean, extensible architecture for future growth
|
||||
|
||||
## Future Provider Addition Example
|
||||
|
||||
When ready to add a new provider (e.g., Alpha Vantage), the process will be:
|
||||
1. Create `ts/stocks/providers/provider.alphavantage.ts`
|
||||
2. Implement the `IStockProvider` interface
|
||||
3. Register the provider in the StockPriceService
|
||||
4. No changes needed to existing code or interfaces
|
||||
42
test/test.handelsregister.ts
Normal file
42
test/test.handelsregister.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js'
|
||||
|
||||
import { BusinessRecord } from '../ts/classes.businessrecord.js';
|
||||
|
||||
let testOpenDataInstance: opendata.OpenData;
|
||||
|
||||
tap.test('first test', async () => {
|
||||
testOpenDataInstance = new opendata.OpenData();
|
||||
expect(testOpenDataInstance).toBeInstanceOf(opendata.OpenData);
|
||||
});
|
||||
|
||||
tap.test('should start the instance', async () => {
|
||||
await testOpenDataInstance.start();
|
||||
});
|
||||
|
||||
const resultsSearch = tap.test('should get the data for a company', async () => {
|
||||
const result = await testOpenDataInstance.handelsregister.searchCompany('LADR', 20);
|
||||
console.log(result);
|
||||
return result;
|
||||
});
|
||||
|
||||
tap.test('should get the data for a specific company', async () => {
|
||||
let testCompany: BusinessRecord['data']['germanParsedRegistration'] = (await resultsSearch.testResultPromise)[0]['germanParsedRegistration'];
|
||||
console.log(`trying to find specific company with:`);
|
||||
console.log(testCompany);
|
||||
const result = await testOpenDataInstance.handelsregister.getSpecificCompany(testCompany);
|
||||
console.log(result);
|
||||
|
||||
await Promise.all(result.files.map(async (file) => {
|
||||
await file.writeToDir('./.nogit/testoutput');
|
||||
}));
|
||||
|
||||
|
||||
});
|
||||
|
||||
tap.test('should stop the instance', async (toolsArg) => {
|
||||
await testOpenDataInstance.stop();
|
||||
});
|
||||
|
||||
|
||||
tap.start()
|
||||
280
test/test.stocks.ts
Normal file
280
test/test.stocks.ts
Normal file
@@ -0,0 +1,280 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js';
|
||||
|
||||
// Test data
|
||||
const testTickers = ['AAPL', 'MSFT', 'GOOGL'];
|
||||
const invalidTicker = 'INVALID_TICKER_XYZ';
|
||||
|
||||
let stockService: opendata.StockPriceService;
|
||||
let yahooProvider: opendata.YahooFinanceProvider;
|
||||
|
||||
tap.test('should create StockPriceService instance', async () => {
|
||||
stockService = new opendata.StockPriceService({
|
||||
ttl: 30000, // 30 seconds cache
|
||||
maxEntries: 100
|
||||
});
|
||||
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
|
||||
});
|
||||
|
||||
tap.test('should create YahooFinanceProvider instance', async () => {
|
||||
yahooProvider = new opendata.YahooFinanceProvider({
|
||||
enabled: true,
|
||||
timeout: 10000,
|
||||
retryAttempts: 2,
|
||||
retryDelay: 500
|
||||
});
|
||||
expect(yahooProvider).toBeInstanceOf(opendata.YahooFinanceProvider);
|
||||
expect(yahooProvider.name).toEqual('Yahoo Finance');
|
||||
expect(yahooProvider.requiresAuth).toEqual(false);
|
||||
});
|
||||
|
||||
tap.test('should register Yahoo provider with the service', async () => {
|
||||
stockService.register(yahooProvider);
|
||||
const providers = stockService.getAllProviders();
|
||||
expect(providers).toContainEqual(yahooProvider);
|
||||
expect(stockService.getProvider('Yahoo Finance')).toEqual(yahooProvider);
|
||||
});
|
||||
|
||||
tap.test('should check provider health', async () => {
|
||||
const health = await stockService.checkProvidersHealth();
|
||||
expect(health.get('Yahoo Finance')).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should fetch single stock price', async () => {
|
||||
const price = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
expect(price).toHaveProperty('ticker');
|
||||
expect(price).toHaveProperty('price');
|
||||
expect(price).toHaveProperty('currency');
|
||||
expect(price).toHaveProperty('change');
|
||||
expect(price).toHaveProperty('changePercent');
|
||||
expect(price).toHaveProperty('previousClose');
|
||||
expect(price).toHaveProperty('timestamp');
|
||||
expect(price).toHaveProperty('provider');
|
||||
expect(price).toHaveProperty('marketState');
|
||||
|
||||
expect(price.ticker).toEqual('AAPL');
|
||||
expect(price.price).toBeGreaterThan(0);
|
||||
expect(price.provider).toEqual('Yahoo Finance');
|
||||
expect(price.timestamp).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
tap.test('should fetch multiple stock prices', async () => {
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: testTickers
|
||||
});
|
||||
|
||||
expect(prices).toBeArray();
|
||||
expect(prices.length).toBeGreaterThan(0);
|
||||
expect(prices.length).toBeLessThanOrEqual(testTickers.length);
|
||||
|
||||
for (const price of prices) {
|
||||
expect(testTickers).toContain(price.ticker);
|
||||
expect(price.price).toBeGreaterThan(0);
|
||||
expect(price.provider).toEqual('Yahoo Finance');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should serve cached prices on subsequent requests', async () => {
|
||||
// First request - should hit the API
|
||||
const firstRequest = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
// Second request - should be served from cache
|
||||
const secondRequest = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
expect(secondRequest.ticker).toEqual(firstRequest.ticker);
|
||||
expect(secondRequest.price).toEqual(firstRequest.price);
|
||||
expect(secondRequest.timestamp).toEqual(firstRequest.timestamp);
|
||||
});
|
||||
|
||||
tap.test('should handle invalid ticker gracefully', async () => {
|
||||
try {
|
||||
await stockService.getPrice({ ticker: invalidTicker });
|
||||
throw new Error('Should have thrown an error for invalid ticker');
|
||||
} catch (error) {
|
||||
expect(error.message).toInclude('Failed to fetch price');
|
||||
expect(error.message).toInclude(invalidTicker);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should support market checking', async () => {
|
||||
expect(yahooProvider.supportsMarket('US')).toEqual(true);
|
||||
expect(yahooProvider.supportsMarket('UK')).toEqual(true);
|
||||
expect(yahooProvider.supportsMarket('DE')).toEqual(true);
|
||||
expect(yahooProvider.supportsMarket('INVALID')).toEqual(false);
|
||||
});
|
||||
|
||||
tap.test('should validate ticker format', async () => {
|
||||
expect(yahooProvider.supportsTicker('AAPL')).toEqual(true);
|
||||
expect(yahooProvider.supportsTicker('MSFT')).toEqual(true);
|
||||
expect(yahooProvider.supportsTicker('BRK.B')).toEqual(true);
|
||||
expect(yahooProvider.supportsTicker('123456789012')).toEqual(false);
|
||||
expect(yahooProvider.supportsTicker('invalid@ticker')).toEqual(false);
|
||||
});
|
||||
|
||||
tap.test('should get provider statistics', async () => {
|
||||
const stats = stockService.getProviderStats();
|
||||
const yahooStats = stats.get('Yahoo Finance');
|
||||
|
||||
expect(yahooStats).not.toEqual(undefined);
|
||||
expect(yahooStats.successCount).toBeGreaterThan(0);
|
||||
expect(yahooStats.errorCount).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should clear cache', async () => {
|
||||
// Ensure we have something in cache
|
||||
await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
// Clear cache
|
||||
stockService.clearCache();
|
||||
|
||||
// Next request should hit the API again (we can't directly test this,
|
||||
// but we can verify the method doesn't throw)
|
||||
const price = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
expect(price).not.toEqual(undefined);
|
||||
});
|
||||
|
||||
tap.test('should handle provider unavailability', async () => {
|
||||
// Clear cache first to ensure we don't get cached results
|
||||
stockService.clearCache();
|
||||
|
||||
// Unregister all providers
|
||||
stockService.unregister('Yahoo Finance');
|
||||
|
||||
try {
|
||||
// Use a different ticker to avoid any caching
|
||||
await stockService.getPrice({ ticker: 'TSLA' });
|
||||
throw new Error('Should have thrown an error with no providers');
|
||||
} catch (error: any) {
|
||||
expect(error.message).toEqual('No stock price providers available');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should fetch major market indicators', async () => {
|
||||
// Re-register provider if needed
|
||||
if (!stockService.getProvider('Yahoo Finance')) {
|
||||
stockService.register(yahooProvider);
|
||||
}
|
||||
|
||||
const marketIndicators = [
|
||||
// Indices
|
||||
{ ticker: '^GSPC', name: 'S&P 500' },
|
||||
{ ticker: '^IXIC', name: 'NASDAQ' },
|
||||
{ ticker: '^DJI', name: 'DOW Jones' },
|
||||
// Tech Stocks
|
||||
{ ticker: 'AAPL', name: 'Apple' },
|
||||
{ ticker: 'AMZN', name: 'Amazon' },
|
||||
{ ticker: 'GOOGL', name: 'Google' },
|
||||
{ ticker: 'META', name: 'Meta' },
|
||||
{ ticker: 'MSFT', name: 'Microsoft' },
|
||||
{ ticker: 'PLTR', name: 'Palantir' },
|
||||
// Crypto
|
||||
{ ticker: 'BTC-USD', name: 'Bitcoin' },
|
||||
{ ticker: 'ETH-USD', name: 'Ethereum' },
|
||||
{ ticker: 'ADA-USD', name: 'Cardano' },
|
||||
// Forex & Commodities
|
||||
{ ticker: 'EURUSD=X', name: 'EUR/USD' },
|
||||
{ ticker: 'GC=F', name: 'Gold Futures' },
|
||||
{ ticker: 'CL=F', name: 'Crude Oil Futures' }
|
||||
];
|
||||
|
||||
console.log('\n📊 Current Market Values:');
|
||||
console.log('═'.repeat(65));
|
||||
|
||||
// Fetch all prices in batch for better performance
|
||||
try {
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: marketIndicators.map(i => i.ticker)
|
||||
});
|
||||
|
||||
// Create a map for easy lookup
|
||||
const priceMap = new Map(prices.map(p => [p.ticker, p]));
|
||||
|
||||
// Check which tickers are missing and fetch them individually
|
||||
const missingTickers: typeof marketIndicators = [];
|
||||
for (const indicator of marketIndicators) {
|
||||
if (!priceMap.has(indicator.ticker)) {
|
||||
missingTickers.push(indicator);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch missing tickers individually
|
||||
if (missingTickers.length > 0) {
|
||||
for (const indicator of missingTickers) {
|
||||
try {
|
||||
const price = await stockService.getPrice({ ticker: indicator.ticker });
|
||||
priceMap.set(indicator.ticker, price);
|
||||
} catch (error) {
|
||||
// Ignore individual errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Display all results with section headers
|
||||
let lastSection = '';
|
||||
for (const indicator of marketIndicators) {
|
||||
// Add section headers
|
||||
if (indicator.ticker.startsWith('^') && lastSection !== 'indices') {
|
||||
console.log('\n📈 Market Indices');
|
||||
console.log('─'.repeat(65));
|
||||
lastSection = 'indices';
|
||||
} else if (['AAPL', 'AMZN', 'GOOGL', 'META', 'MSFT', 'PLTR'].includes(indicator.ticker) && lastSection !== 'stocks') {
|
||||
console.log('\n💻 Tech Stocks');
|
||||
console.log('─'.repeat(65));
|
||||
lastSection = 'stocks';
|
||||
} else if (indicator.ticker.includes('-USD') && lastSection !== 'crypto') {
|
||||
console.log('\n🪙 Cryptocurrencies');
|
||||
console.log('─'.repeat(65));
|
||||
lastSection = 'crypto';
|
||||
} else if ((indicator.ticker.includes('=') || indicator.ticker.includes('=F')) && lastSection !== 'forex') {
|
||||
console.log('\n💱 Forex & Commodities');
|
||||
console.log('─'.repeat(65));
|
||||
lastSection = 'forex';
|
||||
}
|
||||
|
||||
const price = priceMap.get(indicator.ticker);
|
||||
if (price) {
|
||||
const changeSymbol = price.change >= 0 ? '↑' : '↓';
|
||||
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m'; // Green or Red
|
||||
const resetColor = '\x1b[0m';
|
||||
|
||||
console.log(
|
||||
`${indicator.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: indicator.name.includes('coin') || indicator.name.includes('EUR') || indicator.name === 'Cardano' ? 4 : 2
|
||||
}).padStart(12)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
|
||||
);
|
||||
} else {
|
||||
console.log(`${indicator.name.padEnd(20)} Data not available`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('Error fetching market data:', error);
|
||||
// Fallback to individual fetches
|
||||
for (const indicator of marketIndicators) {
|
||||
try {
|
||||
const price = await stockService.getPrice({ ticker: indicator.ticker });
|
||||
const changeSymbol = price.change >= 0 ? '↑' : '↓';
|
||||
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
|
||||
const resetColor = '\x1b[0m';
|
||||
|
||||
console.log(
|
||||
`${indicator.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: indicator.name.includes('coin') || indicator.name.includes('EUR') || indicator.name === 'Cardano' ? 4 : 2
|
||||
}).padStart(12)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.log(`${indicator.name.padEnd(20)} Error fetching data`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('═'.repeat(65));
|
||||
console.log(`Last updated: ${new Date().toLocaleString()}\n`);
|
||||
|
||||
// Test passes if we successfully fetch at least some indicators
|
||||
expect(true).toEqual(true);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
13
test/test.ts
13
test/test.ts
@@ -1,6 +1,8 @@
|
||||
import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js'
|
||||
|
||||
import { BusinessRecord } from '../ts/classes.businessrecord.js';
|
||||
|
||||
let testOpenDataInstance: opendata.OpenData;
|
||||
|
||||
tap.test('first test', async () => {
|
||||
@@ -12,4 +14,13 @@ tap.test('should start the instance', async () => {
|
||||
await testOpenDataInstance.start();
|
||||
})
|
||||
|
||||
tap.test('should build initial data', async () => {
|
||||
await testOpenDataInstance.buildInitialDb();
|
||||
});
|
||||
|
||||
tap.test('should stop the instance', async () => {
|
||||
await testOpenDataInstance.stop();
|
||||
});
|
||||
|
||||
|
||||
tap.start()
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@fin.cx/opendata',
|
||||
version: '1.1.5',
|
||||
description: 'A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.'
|
||||
version: '1.6.0',
|
||||
description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
|
||||
}
|
||||
|
||||
@@ -1,32 +1,67 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
@plugins.smartdata.Manager()
|
||||
export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<BusinessRecord, BusinessRecord> {
|
||||
export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<
|
||||
BusinessRecord,
|
||||
BusinessRecord
|
||||
> {
|
||||
// STATIC
|
||||
public static getByGermanParsedRegistration = async (parsedGermanRegistrationArg: BusinessRecord['data']['germanParsedRegistration']) => {
|
||||
const businessRecords = await BusinessRecord.getInstance({
|
||||
data: {
|
||||
germanParsedRegistration: parsedGermanRegistrationArg,
|
||||
}
|
||||
});
|
||||
return businessRecords;
|
||||
};
|
||||
|
||||
|
||||
// INSTANCE
|
||||
@plugins.smartdata.unI()
|
||||
id: string;
|
||||
|
||||
@plugins.smartdata.svDb()
|
||||
data: {
|
||||
name?: string,
|
||||
address?: string,
|
||||
postalCode?: string,
|
||||
city?: string,
|
||||
country?: string,
|
||||
phone?: string,
|
||||
fax?: string,
|
||||
email?: string,
|
||||
website?: string,
|
||||
businessType?: string,
|
||||
registrationNumber?: string,
|
||||
registrationCourt?: string,
|
||||
legalForm?: string,
|
||||
managingDirectors?: string[],
|
||||
boardOfDirectors?: string[],
|
||||
supervisoryBoard?: string[],
|
||||
foundingDate?: string,
|
||||
capital?: string,
|
||||
purpose?: string,
|
||||
lastUpdate?: string
|
||||
name?: string;
|
||||
startDate?: string;
|
||||
endDate?: string;
|
||||
status?: 'active' | 'liquidating' | 'closed';
|
||||
address?: string;
|
||||
postalCode?: string;
|
||||
city?: string;
|
||||
country?: string;
|
||||
phone?: string;
|
||||
fax?: string;
|
||||
email?: string;
|
||||
website?: string;
|
||||
businessType?: string;
|
||||
registrationId?: string;
|
||||
germanParsedRegistration?: {
|
||||
court?: string;
|
||||
type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR';
|
||||
number?: string;
|
||||
};
|
||||
legalForm?:
|
||||
| 'GmbH'
|
||||
| 'GmbH & Co. KG'
|
||||
| 'AG'
|
||||
| 'LLC'
|
||||
| 'LLP'
|
||||
| 'GmbH & Co. KGaA'
|
||||
| 'GmbH & Co. KGaA, LLC';
|
||||
managingDirectors?: string[];
|
||||
boardOfDirectors?: string[];
|
||||
supervisoryBoard?: string[];
|
||||
foundingDate?: string;
|
||||
capital?: string;
|
||||
purpose?: string;
|
||||
lastUpdate?: string;
|
||||
} = {};
|
||||
|
||||
/**
|
||||
* validates the record against the Handelregister.
|
||||
*/
|
||||
public async validate() {
|
||||
if (!this.data.name) throw new Error('Name is required.');
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import type { OpenData } from './classes.main.opendata.js';
|
||||
|
||||
export class GermanBusinessData {
|
||||
public openDataRef: OpenData;
|
||||
constructor(openDataRefArg: OpenData) {
|
||||
this.openDataRef = openDataRefArg;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
await this.update();
|
||||
}
|
||||
public async stop() {}
|
||||
|
||||
public async update() {
|
||||
const done = plugins.smartpromise.defer();
|
||||
const promiseArray: Promise<any>[] = [];
|
||||
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||||
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
|
||||
if (!dataExists) {
|
||||
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
|
||||
} else {
|
||||
}
|
||||
|
||||
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
||||
promiseArray
|
||||
.push
|
||||
// smartarchive.exportToFs(paths.germanBusinessDataDir, 'de_companies_ocdata.jsonl')
|
||||
();
|
||||
const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
|
||||
let totalRecordsCounter = 0;
|
||||
let nextRest: string = '';
|
||||
jsonlDataStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
|
||||
const readStream = await chunkArg.createReadStream();
|
||||
readStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
|
||||
const currentString = nextRest + chunkArg.toString();
|
||||
const lines = currentString.split('\n');
|
||||
nextRest = lines.pop();
|
||||
console.log(`Got another ${lines.length} records.`);
|
||||
for (const line of lines) {
|
||||
let entry: any;
|
||||
if (!line) continue;
|
||||
try {
|
||||
entry = JSON.parse(line);
|
||||
} catch (err) {
|
||||
console.log(line);
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
}
|
||||
if (!entry) continue;
|
||||
totalRecordsCounter++;
|
||||
if (totalRecordsCounter % 10000 === 0) console.log(`${totalRecordsCounter} total records.`);
|
||||
const businessRecord = new this.openDataRef.CBusinessRecord();
|
||||
businessRecord.id = await this.openDataRef.CBusinessRecord.getNewId();
|
||||
businessRecord.data.name = entry.name;
|
||||
await businessRecord.save();
|
||||
// console.log(`stored ${businessRecord.data.name}`);
|
||||
}
|
||||
},
|
||||
finalFunction: async (streamToolsArg) => {
|
||||
console.log(`finished processing ${totalRecordsCounter} records.`);
|
||||
if (!nextRest) return;
|
||||
JSON.parse(nextRest);
|
||||
}
|
||||
})
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
public async getBusinessRecordByName(nameArg: string) {
|
||||
const businessRecord = await this.openDataRef.CBusinessRecord.getInstance({
|
||||
data: {
|
||||
name: { $regex: `${nameArg}`, $options: "i" } as any,
|
||||
}
|
||||
});
|
||||
return businessRecord;
|
||||
}
|
||||
}
|
||||
358
ts/classes.handelsregister.ts
Normal file
358
ts/classes.handelsregister.ts
Normal file
@@ -0,0 +1,358 @@
|
||||
import type { BusinessRecord } from './classes.businessrecord.js';
|
||||
import type { OpenData } from './classes.main.opendata.js';
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
|
||||
/**
|
||||
* the HandlesRegister exposed as a class
|
||||
*/
|
||||
export class HandelsRegister {
|
||||
private openDataRef: OpenData;
|
||||
private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
|
||||
private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple());
|
||||
|
||||
// Puppeteer wrapper instance
|
||||
public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
|
||||
|
||||
constructor(openDataRef: OpenData) {
|
||||
this.openDataRef = openDataRef;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
// Start the browser
|
||||
await plugins.smartfile.fs.ensureDir(this.uniqueDowloadFolder);
|
||||
await this.smartbrowserInstance.start();
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
// Stop the browser
|
||||
await plugins.smartfile.fs.remove(this.uniqueDowloadFolder);
|
||||
await this.smartbrowserInstance.stop();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new page and configures it to allow file downloads
|
||||
* to a predefined path.
|
||||
*/
|
||||
public getNewPage = async () => {
|
||||
const page = await this.smartbrowserInstance.headlessBrowser.newPage();
|
||||
|
||||
// 1) Create a DevTools session for this page
|
||||
const cdpSession = await page.target().createCDPSession();
|
||||
|
||||
// 2) Allow file downloads and set the download path
|
||||
await cdpSession.send('Page.setDownloadBehavior', {
|
||||
behavior: 'allow',
|
||||
downloadPath: this.uniqueDowloadFolder, // <-- Change this to your desired absolute path
|
||||
});
|
||||
|
||||
// Optionally set viewport and go to page
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
await page.goto('https://www.handelsregister.de/');
|
||||
return page;
|
||||
};
|
||||
|
||||
private navigateToPage = async (
|
||||
pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
|
||||
pageNameArg: string
|
||||
) => {
|
||||
try {
|
||||
await pageArg.evaluate((pageNameArg2) => {
|
||||
const elements = Array.from(document.querySelectorAll('.ui-menuitem-text > span'));
|
||||
const targetElement = elements.find((el) => el.textContent?.trim() === pageNameArg2);
|
||||
if (targetElement) {
|
||||
(targetElement as HTMLElement).click();
|
||||
}
|
||||
}, pageNameArg);
|
||||
console.log(`Navigated to the ${pageNameArg} page successfully.`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to navigate to the ${pageNameArg} page:`, error);
|
||||
}
|
||||
};
|
||||
|
||||
private waitForResults = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page) => {
|
||||
await pageArg
|
||||
.waitForSelector('#ergebnissForm\\:selectedSuchErgebnisFormTable_data', {
|
||||
timeout: 30000,
|
||||
})
|
||||
.catch(async (err) => {
|
||||
await pageArg.screenshot({ path: paths.downloadDir + '/error.png' });
|
||||
throw err;
|
||||
});
|
||||
|
||||
const businessRecords: BusinessRecord['data'][] = await pageArg.evaluate(() => {
|
||||
const rows = document.querySelectorAll(
|
||||
'#ergebnissForm\\:selectedSuchErgebnisFormTable_data > tr'
|
||||
);
|
||||
const records: BusinessRecord['data'][] = [];
|
||||
|
||||
rows.forEach((row) => {
|
||||
const nameElement = row.querySelector('td.ui-panelgrid-cell span.marginLeft20');
|
||||
const cityElement = row.querySelector('td.ui-panelgrid-cell.sitzSuchErgebnisse span');
|
||||
const statusElement = row.querySelector('td.ui-panelgrid-cell span.verticalText');
|
||||
const registrationCourtElement = row.querySelector(
|
||||
'td.ui-panelgrid-cell.fontTableNameSize'
|
||||
);
|
||||
|
||||
const name = nameElement?.textContent?.trim();
|
||||
const city = cityElement?.textContent?.trim();
|
||||
const status = statusElement?.textContent?.trim();
|
||||
const registrationId = registrationCourtElement?.textContent?.trim();
|
||||
|
||||
// Push parsed data into records array
|
||||
records.push({
|
||||
name,
|
||||
city,
|
||||
registrationId,
|
||||
businessType: status,
|
||||
});
|
||||
});
|
||||
|
||||
return records;
|
||||
});
|
||||
return businessRecords;
|
||||
};
|
||||
|
||||
private clickFindButton = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page, resultsLimitArg: number = 100) => {
|
||||
try {
|
||||
// Wait for the button with the text "Find" to appear
|
||||
await pageArg.waitForSelector('span.ui-button-text.ui-c', { timeout: 5000 });
|
||||
|
||||
// adjust to 100 results per page
|
||||
await pageArg.select('#form\\:ergebnisseProSeite_input', `${resultsLimitArg}`);
|
||||
|
||||
// Locate and click the button using its text
|
||||
await pageArg.evaluate(() => {
|
||||
const buttons = Array.from(document.querySelectorAll('span.ui-button-text.ui-c'));
|
||||
const targetButton = buttons.find((button) => button.textContent?.trim() === 'Find');
|
||||
if (targetButton) {
|
||||
const parentButton = targetButton.closest('button') || targetButton;
|
||||
(parentButton as HTMLElement).click();
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Find button clicked successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or click the "Find" button:', error);
|
||||
}
|
||||
};
|
||||
|
||||
private async downloadFile(
|
||||
pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
|
||||
typeArg: 'SI' | 'AD'
|
||||
) {
|
||||
// Trigger the file download by clicking on the relevant link
|
||||
await pageArg.evaluate((typeArg2) => {
|
||||
// Locate the table body
|
||||
const tableBody = document.querySelector(
|
||||
'#ergebnissForm\\:selectedSuchErgebnisFormTable_data'
|
||||
);
|
||||
if (!tableBody) {
|
||||
throw new Error('Table body not found');
|
||||
}
|
||||
|
||||
// Locate the first row
|
||||
const firstRow = tableBody.querySelector('tr:nth-child(1)');
|
||||
if (!firstRow) {
|
||||
throw new Error('First row not found');
|
||||
}
|
||||
|
||||
// Locate the last cell in the first row
|
||||
const lastCell = firstRow.querySelector('td:last-child');
|
||||
if (!lastCell) {
|
||||
throw new Error('Last cell not found in the first row');
|
||||
}
|
||||
|
||||
// Locate the download links
|
||||
const adLink = lastCell.querySelector('a:first-of-type');
|
||||
const siLink = lastCell.querySelector('a:last-of-type');
|
||||
if (!siLink) {
|
||||
throw new Error('SI link not found in the last cell');
|
||||
}
|
||||
|
||||
// Simulate a click on the last <a> element
|
||||
switch (typeArg2) {
|
||||
case 'AD':
|
||||
(adLink as HTMLElement).click();
|
||||
break;
|
||||
case 'SI':
|
||||
(siLink as HTMLElement).click();
|
||||
break;
|
||||
default:
|
||||
throw new Error('Invalid file type');
|
||||
}
|
||||
}, typeArg);
|
||||
|
||||
|
||||
await plugins.smartfile.fs.waitForFileToBeReady(this.uniqueDowloadFolder);
|
||||
|
||||
const files = await plugins.smartfile.fs.fileTreeToObject(this.uniqueDowloadFolder, '**/*');
|
||||
const file = files[0];
|
||||
|
||||
// lets clear the folder for the next download
|
||||
await plugins.smartfile.fs.ensureEmptyDir(this.uniqueDowloadFolder);
|
||||
switch (typeArg) {
|
||||
case 'AD':
|
||||
await file.rename(`ad.pdf`);
|
||||
break;
|
||||
case 'SI':
|
||||
await file.rename(`si.xml`);
|
||||
break;
|
||||
break;
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to parse the German registration string
|
||||
*/
|
||||
private async parseGermanRegistration(
|
||||
input: string
|
||||
): Promise<BusinessRecord['data']['germanParsedRegistration']> {
|
||||
// e.g. District court Berlin (Charlottenburg) HRB 123456
|
||||
const regex =
|
||||
/District court (\p{L}[\p{L}\s-]*?(?:\s*\([\p{L}\s-]+\))?)\s+(HRA|HRB|GnR|VR|PR|GsR)\s+(\d+)/u;
|
||||
const match = input.match(regex);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
court: match[1],
|
||||
type: match[2] as 'HRA' | 'HRB', // Adjust if needed
|
||||
number: match[3],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for a company by name and return basic info
|
||||
*/
|
||||
public async searchCompany(companyNameArg: string, resultsLimitArg: number = 100) {
|
||||
return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const page = await this.getNewPage();
|
||||
await this.navigateToPage(page, 'Normal search');
|
||||
|
||||
try {
|
||||
// Wait for the textarea to appear
|
||||
await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
|
||||
|
||||
// Enter text into the textarea
|
||||
const inputText = companyNameArg;
|
||||
await page.evaluate((text) => {
|
||||
const textarea = document.querySelector<HTMLTextAreaElement>('#form\\:schlagwoerter');
|
||||
if (textarea) {
|
||||
textarea.value = text; // Set the value
|
||||
// Trigger the change event manually if required
|
||||
const event = new Event('change', { bubbles: true });
|
||||
textarea.dispatchEvent(event);
|
||||
}
|
||||
}, inputText);
|
||||
|
||||
console.log('Text entered successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or enter text into the textarea:', error);
|
||||
}
|
||||
|
||||
try {
|
||||
// Wait for the radio button's label to appear
|
||||
await page.waitForSelector('label[for="form:schlagwortOptionen:0"]', { timeout: 5000 });
|
||||
|
||||
// Click the label to select the radio button
|
||||
await page.evaluate(() => {
|
||||
const label = document.querySelector<HTMLLabelElement>(
|
||||
'label[for="form:schlagwortOptionen:0"]'
|
||||
);
|
||||
if (label) {
|
||||
label.click();
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Radio button clicked successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or click the radio button:', error);
|
||||
}
|
||||
|
||||
await this.clickFindButton(page, resultsLimitArg);
|
||||
|
||||
const businessRecords = await this.waitForResults(page);
|
||||
|
||||
// Parse out the registration info
|
||||
for (const record of businessRecords) {
|
||||
if (record.registrationId) {
|
||||
record.germanParsedRegistration = await this.parseGermanRegistration(
|
||||
record.registrationId
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await page.close();
|
||||
return businessRecords;
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for a specific company (known register type/number/court),
|
||||
* then click on an element that triggers a file download.
|
||||
*/
|
||||
public async getSpecificCompany(companyArg: BusinessRecord['data']['germanParsedRegistration']) {
|
||||
return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
|
||||
const page = await this.getNewPage();
|
||||
await this.navigateToPage(page, 'Normal search');
|
||||
await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
|
||||
|
||||
// 1) Type of Register (e.g. HRB, HRA, etc.)
|
||||
await page.waitForSelector('#form\\:registerArt_label');
|
||||
await page.click('#form\\:registerArt_label');
|
||||
await page.waitForSelector('#form\\:registerArt_items');
|
||||
await page.evaluate((type) => {
|
||||
const options = Array.from(document.querySelectorAll('#form\\:registerArt_items li'));
|
||||
const targetOption = options.find((option) => option.textContent?.trim() === type);
|
||||
(targetOption as HTMLElement)?.click();
|
||||
}, companyArg.type);
|
||||
|
||||
// 2) Register number
|
||||
await page.waitForSelector('#form\\:registerNummer');
|
||||
await page.type('#form\\:registerNummer', companyArg.number);
|
||||
|
||||
// 3) Register court
|
||||
await page.waitForSelector('#form\\:registergericht_label');
|
||||
await page.click('#form\\:registergericht_label');
|
||||
await page.waitForSelector('#form\\:registergericht_items');
|
||||
await page.evaluate((court) => {
|
||||
const options = Array.from(document.querySelectorAll('#form\\:registergericht_items li'));
|
||||
const targetOption = options.find((option) => option.textContent?.trim() === court);
|
||||
(targetOption as HTMLElement)?.click();
|
||||
}, companyArg.court);
|
||||
|
||||
// Click 'Find'
|
||||
await this.clickFindButton(page);
|
||||
|
||||
// Optionally grab the results, just for logging
|
||||
const businessRecords = await this.waitForResults(page);
|
||||
console.log(businessRecords);
|
||||
|
||||
const files: plugins.smartfile.SmartFile[] = [];
|
||||
|
||||
// download files
|
||||
files.push(await this.downloadFile(page, 'SI'));
|
||||
files.push(await this.downloadFile(page, 'AD'));
|
||||
|
||||
// At this point, the file should have been downloaded automatically
|
||||
// to the path specified by `Page.setDownloadBehavior`
|
||||
await page.close();
|
||||
|
||||
return {
|
||||
businessRecords,
|
||||
files,
|
||||
};
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
/**
|
||||
* get specific company by full name
|
||||
*/
|
||||
public async getSpecificCompanyByName(companyNameArg: string) {
|
||||
const businessRecords = await this.searchCompany(companyNameArg, 1);
|
||||
const result = this.getSpecificCompany(businessRecords[0].germanParsedRegistration);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
111
ts/classes.jsonldata.ts
Normal file
111
ts/classes.jsonldata.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import type { OpenData } from './classes.main.opendata.js';
|
||||
|
||||
export type SeedEntryType = {
|
||||
all_attributes: {
|
||||
_registerArt: string;
|
||||
_registerNummer: string;
|
||||
additional_data: {
|
||||
AD: boolean;
|
||||
CD: boolean;
|
||||
DK: boolean;
|
||||
HD: boolean;
|
||||
SI: boolean;
|
||||
UT: boolean;
|
||||
VÖ: boolean;
|
||||
};
|
||||
federal_state: string;
|
||||
native_company_number: string;
|
||||
registered_office: string;
|
||||
registrar: string;
|
||||
};
|
||||
company_number: string;
|
||||
current_status: string;
|
||||
jurisdiction_code: string;
|
||||
name: string;
|
||||
officers: {
|
||||
name: string;
|
||||
other_attributes: {
|
||||
city: string;
|
||||
firstname: string;
|
||||
flag: string;
|
||||
lastname: string;
|
||||
};
|
||||
position: string;
|
||||
start_date: string; // ISO 8601 date string
|
||||
type: string;
|
||||
}[];
|
||||
registered_address: string;
|
||||
retrieved_at: string; // ISO 8601 date string
|
||||
};
|
||||
|
||||
export class JsonlDataProcessor<T> {
|
||||
public forEachFunction: (entryArg: T) => Promise<void>;
|
||||
constructor(forEachFunctionArg: typeof this.forEachFunction) {
|
||||
this.forEachFunction = forEachFunctionArg;
|
||||
}
|
||||
|
||||
// TODO: define a mapper as argument instead of hard-coding it
|
||||
public async processDataFromUrl(
|
||||
dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
|
||||
) {
|
||||
const done = plugins.smartpromise.defer();
|
||||
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
|
||||
if (!dataExists) {
|
||||
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
|
||||
} else {
|
||||
}
|
||||
|
||||
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
|
||||
const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
|
||||
let totalRecordsCounter = 0;
|
||||
let nextRest: string = '';
|
||||
jsonlDataStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
|
||||
const readStream = await chunkArg.createReadStream();
|
||||
readStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
|
||||
const currentString = nextRest + chunkArg.toString();
|
||||
const lines = currentString.split('\n');
|
||||
nextRest = lines.pop();
|
||||
console.log(`Got another ${lines.length} records.`);
|
||||
const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor<string>(
|
||||
async (line) => {
|
||||
let entry: T;
|
||||
if (!line) return;
|
||||
try {
|
||||
entry = JSON.parse(line);
|
||||
} catch (err) {
|
||||
console.log(line);
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
}
|
||||
if (!entry) return;
|
||||
totalRecordsCounter++;
|
||||
if (totalRecordsCounter % 10000 === 0)
|
||||
console.log(`${totalRecordsCounter} total records.`);
|
||||
await this.forEachFunction(entry);
|
||||
},
|
||||
1000
|
||||
);
|
||||
await concurrentProcessor.process(lines);
|
||||
},
|
||||
finalFunction: async (streamToolsArg) => {
|
||||
console.log(`finished processing ${totalRecordsCounter} records.`);
|
||||
if (nextRest) {
|
||||
JSON.parse(nextRest);
|
||||
};
|
||||
done.resolve();
|
||||
},
|
||||
})
|
||||
);
|
||||
},
|
||||
})
|
||||
);
|
||||
await done.promise;
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,61 @@
|
||||
import { BusinessRecord } from './classes.businessrecord.js';
|
||||
import { GermanBusinessData } from './classes.germanbusinessdata.js';
|
||||
import { HandelsRegister } from './classes.handelsregister.js';
|
||||
import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
|
||||
import * as paths from './paths.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export class OpenData {
|
||||
db: plugins.smartdata.SmartdataDb;
|
||||
germanBusinesses: GermanBusinessData;
|
||||
public db: plugins.smartdata.SmartdataDb;
|
||||
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
|
||||
|
||||
public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
|
||||
public handelsregister: HandelsRegister;
|
||||
|
||||
public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
|
||||
|
||||
public async start() {
|
||||
this.db = new plugins.smartdata.SmartdataDb({
|
||||
mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
|
||||
mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
|
||||
mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
|
||||
mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
|
||||
mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
|
||||
mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
|
||||
mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
|
||||
mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
|
||||
});
|
||||
await this.db.init();
|
||||
this.germanBusinesses = new GermanBusinessData(this);
|
||||
await this.germanBusinesses.start();
|
||||
this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => {
|
||||
const businessRecord = new this.CBusinessRecord();
|
||||
businessRecord.id = await this.CBusinessRecord.getNewId();
|
||||
businessRecord.data.name = entryArg.name;
|
||||
businessRecord.data.germanParsedRegistration = {
|
||||
court: entryArg.all_attributes.registered_office,
|
||||
number: entryArg.all_attributes._registerNummer,
|
||||
type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
|
||||
};
|
||||
await businessRecord.save();
|
||||
});
|
||||
this.handelsregister = new HandelsRegister(this);
|
||||
await this.handelsregister.start();
|
||||
}
|
||||
|
||||
public async buildInitialDb() {
|
||||
await this.jsonLDataProcessor.processDataFromUrl();
|
||||
}
|
||||
|
||||
public async slowValidateDb() {
|
||||
|
||||
}
|
||||
|
||||
public async validateSearchByName() {
|
||||
|
||||
}
|
||||
|
||||
public async searchDbByBusinessNameAndPostalCode(businessNameArg: string, postalCodeArg: string) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public async stop() {
|
||||
await this.db.close();
|
||||
await this.handelsregister.stop();
|
||||
}
|
||||
public async stop() {}
|
||||
}
|
||||
@@ -1 +1,2 @@
|
||||
export * from './classes.main.opendata.js';
|
||||
export * from './stocks/index.js';
|
||||
|
||||
@@ -8,4 +8,8 @@ export const packageDir = plugins.path.join(
|
||||
export const nogitDir = plugins.path.join(packageDir, './.nogit/');
|
||||
plugins.smartfile.fs.ensureDirSync(nogitDir);
|
||||
|
||||
export const downloadDir = plugins.path.join(nogitDir, 'downloads');
|
||||
plugins.smartfile.fs.ensureDirSync(downloadDir);
|
||||
|
||||
|
||||
export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');
|
||||
@@ -6,24 +6,43 @@ export {
|
||||
}
|
||||
|
||||
// @push.rocks scope
|
||||
import * as lik from '@push.rocks/lik';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartarchive from '@push.rocks/smartarchive';
|
||||
import * as smartarray from '@push.rocks/smartarray';
|
||||
import * as smartbrowser from '@push.rocks/smartbrowser';
|
||||
import * as smartdata from '@push.rocks/smartdata';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartlog from '@push.rocks/smartlog';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrequest from '@push.rocks/smartrequest';
|
||||
import * as smartstream from '@push.rocks/smartstream';
|
||||
import * as smartunique from '@push.rocks/smartunique';
|
||||
import * as smartxml from '@push.rocks/smartxml';
|
||||
|
||||
export {
|
||||
lik,
|
||||
qenv,
|
||||
smartarchive,
|
||||
smartarray,
|
||||
smartbrowser,
|
||||
smartdata,
|
||||
smartdelay,
|
||||
smartfile,
|
||||
smartlog,
|
||||
smartpath,
|
||||
smartpromise,
|
||||
smartrequest,
|
||||
smartstream,
|
||||
smartunique,
|
||||
smartxml,
|
||||
}
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export {
|
||||
tsclass,
|
||||
}
|
||||
309
ts/stocks/classes.stockservice.ts
Normal file
309
ts/stocks/classes.stockservice.ts
Normal file
@@ -0,0 +1,309 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import type { IStockProvider, IProviderConfig, IProviderRegistry } from './interfaces/provider.js';
|
||||
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest, IStockPriceError } from './interfaces/stockprice.js';
|
||||
|
||||
interface IProviderEntry {
|
||||
provider: IStockProvider;
|
||||
config: IProviderConfig;
|
||||
lastError?: Error;
|
||||
lastErrorTime?: Date;
|
||||
successCount: number;
|
||||
errorCount: number;
|
||||
}
|
||||
|
||||
interface ICacheEntry {
|
||||
price: IStockPrice;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
export class StockPriceService implements IProviderRegistry {
|
||||
private providers = new Map<string, IProviderEntry>();
|
||||
private cache = new Map<string, ICacheEntry>();
|
||||
private logger = console;
|
||||
|
||||
private cacheConfig = {
|
||||
ttl: 60000, // 60 seconds default
|
||||
maxEntries: 1000
|
||||
};
|
||||
|
||||
constructor(cacheConfig?: { ttl?: number; maxEntries?: number }) {
|
||||
if (cacheConfig) {
|
||||
this.cacheConfig = { ...this.cacheConfig, ...cacheConfig };
|
||||
}
|
||||
}
|
||||
|
||||
public register(provider: IStockProvider, config?: IProviderConfig): void {
|
||||
const defaultConfig: IProviderConfig = {
|
||||
enabled: true,
|
||||
priority: provider.priority,
|
||||
timeout: 10000,
|
||||
retryAttempts: 2,
|
||||
retryDelay: 1000
|
||||
};
|
||||
|
||||
const mergedConfig = { ...defaultConfig, ...config };
|
||||
|
||||
this.providers.set(provider.name, {
|
||||
provider,
|
||||
config: mergedConfig,
|
||||
successCount: 0,
|
||||
errorCount: 0
|
||||
});
|
||||
|
||||
console.log(`Registered provider: ${provider.name}`);
|
||||
}
|
||||
|
||||
public unregister(providerName: string): void {
|
||||
this.providers.delete(providerName);
|
||||
console.log(`Unregistered provider: ${providerName}`);
|
||||
}
|
||||
|
||||
public getProvider(name: string): IStockProvider | undefined {
|
||||
return this.providers.get(name)?.provider;
|
||||
}
|
||||
|
||||
public getAllProviders(): IStockProvider[] {
|
||||
return Array.from(this.providers.values()).map(entry => entry.provider);
|
||||
}
|
||||
|
||||
public getEnabledProviders(): IStockProvider[] {
|
||||
return Array.from(this.providers.values())
|
||||
.filter(entry => entry.config.enabled)
|
||||
.sort((a, b) => (b.config.priority || 0) - (a.config.priority || 0))
|
||||
.map(entry => entry.provider);
|
||||
}
|
||||
|
||||
public async getPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
const cacheKey = this.getCacheKey(request);
|
||||
const cached = this.getFromCache(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
console.log(`Cache hit for ${request.ticker}`);
|
||||
return cached;
|
||||
}
|
||||
|
||||
const providers = this.getEnabledProviders();
|
||||
if (providers.length === 0) {
|
||||
throw new Error('No stock price providers available');
|
||||
}
|
||||
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (const provider of providers) {
|
||||
const entry = this.providers.get(provider.name)!;
|
||||
|
||||
try {
|
||||
const price = await this.fetchWithRetry(
|
||||
() => provider.fetchPrice(request),
|
||||
entry.config
|
||||
);
|
||||
|
||||
entry.successCount++;
|
||||
this.addToCache(cacheKey, price);
|
||||
console.log(`Successfully fetched ${request.ticker} from ${provider.name}`);
|
||||
return price;
|
||||
} catch (error) {
|
||||
entry.errorCount++;
|
||||
entry.lastError = error as Error;
|
||||
entry.lastErrorTime = new Date();
|
||||
lastError = error as Error;
|
||||
|
||||
console.warn(
|
||||
`Provider ${provider.name} failed for ${request.ticker}: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Failed to fetch price for ${request.ticker} from all providers. Last error: ${lastError?.message}`
|
||||
);
|
||||
}
|
||||
|
||||
public async getPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
|
||||
const cachedPrices: IStockPrice[] = [];
|
||||
const tickersToFetch: string[] = [];
|
||||
|
||||
// Check cache for each ticker
|
||||
for (const ticker of request.tickers) {
|
||||
const cacheKey = this.getCacheKey({ ticker, includeExtendedHours: request.includeExtendedHours });
|
||||
const cached = this.getFromCache(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
cachedPrices.push(cached);
|
||||
} else {
|
||||
tickersToFetch.push(ticker);
|
||||
}
|
||||
}
|
||||
|
||||
if (tickersToFetch.length === 0) {
|
||||
console.log(`All ${request.tickers.length} tickers served from cache`);
|
||||
return cachedPrices;
|
||||
}
|
||||
|
||||
const providers = this.getEnabledProviders();
|
||||
if (providers.length === 0) {
|
||||
throw new Error('No stock price providers available');
|
||||
}
|
||||
|
||||
let lastError: Error | undefined;
|
||||
let fetchedPrices: IStockPrice[] = [];
|
||||
|
||||
for (const provider of providers) {
|
||||
const entry = this.providers.get(provider.name)!;
|
||||
|
||||
try {
|
||||
fetchedPrices = await this.fetchWithRetry(
|
||||
() => provider.fetchPrices({
|
||||
tickers: tickersToFetch,
|
||||
includeExtendedHours: request.includeExtendedHours
|
||||
}),
|
||||
entry.config
|
||||
);
|
||||
|
||||
entry.successCount++;
|
||||
|
||||
// Cache the fetched prices
|
||||
for (const price of fetchedPrices) {
|
||||
const cacheKey = this.getCacheKey({
|
||||
ticker: price.ticker,
|
||||
includeExtendedHours: request.includeExtendedHours
|
||||
});
|
||||
this.addToCache(cacheKey, price);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Successfully fetched ${fetchedPrices.length} prices from ${provider.name}`
|
||||
);
|
||||
break;
|
||||
} catch (error) {
|
||||
entry.errorCount++;
|
||||
entry.lastError = error as Error;
|
||||
entry.lastErrorTime = new Date();
|
||||
lastError = error as Error;
|
||||
|
||||
console.warn(
|
||||
`Provider ${provider.name} failed for batch request: ${error.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (fetchedPrices.length === 0 && lastError) {
|
||||
throw new Error(
|
||||
`Failed to fetch prices from all providers. Last error: ${lastError.message}`
|
||||
);
|
||||
}
|
||||
|
||||
return [...cachedPrices, ...fetchedPrices];
|
||||
}
|
||||
|
||||
public async checkProvidersHealth(): Promise<Map<string, boolean>> {
|
||||
const health = new Map<string, boolean>();
|
||||
|
||||
for (const [name, entry] of this.providers) {
|
||||
if (!entry.config.enabled) {
|
||||
health.set(name, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const isAvailable = await entry.provider.isAvailable();
|
||||
health.set(name, isAvailable);
|
||||
} catch (error) {
|
||||
health.set(name, false);
|
||||
console.error(`Health check failed for ${name}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return health;
|
||||
}
|
||||
|
||||
public getProviderStats(): Map<string, {
|
||||
successCount: number;
|
||||
errorCount: number;
|
||||
lastError?: string;
|
||||
lastErrorTime?: Date;
|
||||
}> {
|
||||
const stats = new Map();
|
||||
|
||||
for (const [name, entry] of this.providers) {
|
||||
stats.set(name, {
|
||||
successCount: entry.successCount,
|
||||
errorCount: entry.errorCount,
|
||||
lastError: entry.lastError?.message,
|
||||
lastErrorTime: entry.lastErrorTime
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
public clearCache(): void {
|
||||
this.cache.clear();
|
||||
console.log('Cache cleared');
|
||||
}
|
||||
|
||||
public setCacheTTL(ttl: number): void {
|
||||
this.cacheConfig.ttl = ttl;
|
||||
console.log(`Cache TTL set to ${ttl}ms`);
|
||||
}
|
||||
|
||||
private async fetchWithRetry<T>(
|
||||
fetchFn: () => Promise<T>,
|
||||
config: IProviderConfig
|
||||
): Promise<T> {
|
||||
const maxAttempts = config.retryAttempts || 1;
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fetchFn();
|
||||
} catch (error) {
|
||||
lastError = error as Error;
|
||||
|
||||
if (attempt < maxAttempts) {
|
||||
const delay = (config.retryDelay || 1000) * attempt;
|
||||
console.log(`Retry attempt ${attempt} after ${delay}ms`);
|
||||
await plugins.smartdelay.delayFor(delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Unknown error during fetch');
|
||||
}
|
||||
|
||||
private getCacheKey(request: IStockQuoteRequest): string {
|
||||
return `${request.ticker}:${request.includeExtendedHours || false}`;
|
||||
}
|
||||
|
||||
private getFromCache(key: string): IStockPrice | null {
|
||||
const entry = this.cache.get(key);
|
||||
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const age = Date.now() - entry.timestamp.getTime();
|
||||
if (age > this.cacheConfig.ttl) {
|
||||
this.cache.delete(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return entry.price;
|
||||
}
|
||||
|
||||
private addToCache(key: string, price: IStockPrice): void {
|
||||
// Enforce max entries limit
|
||||
if (this.cache.size >= this.cacheConfig.maxEntries) {
|
||||
// Remove oldest entry
|
||||
const oldestKey = this.cache.keys().next().value;
|
||||
if (oldestKey) {
|
||||
this.cache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
price,
|
||||
timestamp: new Date()
|
||||
});
|
||||
}
|
||||
}
|
||||
9
ts/stocks/index.ts
Normal file
9
ts/stocks/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
// Export all interfaces
|
||||
export * from './interfaces/stockprice.js';
|
||||
export * from './interfaces/provider.js';
|
||||
|
||||
// Export main service
|
||||
export * from './classes.stockservice.js';
|
||||
|
||||
// Export providers
|
||||
export * from './providers/provider.yahoo.js';
|
||||
36
ts/stocks/interfaces/provider.ts
Normal file
36
ts/stocks/interfaces/provider.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest } from './stockprice.js';
|
||||
|
||||
export interface IStockProvider {
|
||||
name: string;
|
||||
priority: number;
|
||||
|
||||
fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice>;
|
||||
fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]>;
|
||||
isAvailable(): Promise<boolean>;
|
||||
|
||||
supportsMarket?(market: string): boolean;
|
||||
supportsTicker?(ticker: string): boolean;
|
||||
|
||||
readonly requiresAuth: boolean;
|
||||
readonly rateLimit?: {
|
||||
requestsPerMinute: number;
|
||||
requestsPerDay?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface IProviderConfig {
|
||||
enabled: boolean;
|
||||
priority?: number;
|
||||
apiKey?: string;
|
||||
timeout?: number;
|
||||
retryAttempts?: number;
|
||||
retryDelay?: number;
|
||||
}
|
||||
|
||||
export interface IProviderRegistry {
|
||||
register(provider: IStockProvider, config?: IProviderConfig): void;
|
||||
unregister(providerName: string): void;
|
||||
getProvider(name: string): IStockProvider | undefined;
|
||||
getAllProviders(): IStockProvider[];
|
||||
getEnabledProviders(): IStockProvider[];
|
||||
}
|
||||
30
ts/stocks/interfaces/stockprice.ts
Normal file
30
ts/stocks/interfaces/stockprice.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
export interface IStockPrice {
|
||||
ticker: string;
|
||||
price: number;
|
||||
currency: string;
|
||||
change: number;
|
||||
changePercent: number;
|
||||
previousClose: number;
|
||||
timestamp: Date;
|
||||
provider: string;
|
||||
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
|
||||
exchange?: string;
|
||||
exchangeName?: string;
|
||||
}
|
||||
|
||||
export interface IStockPriceError {
|
||||
ticker: string;
|
||||
error: string;
|
||||
provider: string;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
export interface IStockQuoteRequest {
|
||||
ticker: string;
|
||||
includeExtendedHours?: boolean;
|
||||
}
|
||||
|
||||
export interface IStockBatchQuoteRequest {
|
||||
tickers: string[];
|
||||
includeExtendedHours?: boolean;
|
||||
}
|
||||
159
ts/stocks/providers/provider.yahoo.ts
Normal file
159
ts/stocks/providers/provider.yahoo.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
|
||||
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest } from '../interfaces/stockprice.js';
|
||||
|
||||
export class YahooFinanceProvider implements IStockProvider {
|
||||
public name = 'Yahoo Finance';
|
||||
public priority = 100;
|
||||
public readonly requiresAuth = false;
|
||||
public readonly rateLimit = {
|
||||
requestsPerMinute: 100, // Conservative estimate
|
||||
requestsPerDay: undefined
|
||||
};
|
||||
|
||||
private logger = console;
|
||||
private baseUrl = 'https://query1.finance.yahoo.com';
|
||||
private userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
constructor(private config?: IProviderConfig) {}
|
||||
|
||||
public async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
try {
|
||||
const url = `${this.baseUrl}/v8/finance/chart/${request.ticker}`;
|
||||
const response = await plugins.smartrequest.getJson(url, {
|
||||
headers: {
|
||||
'User-Agent': this.userAgent
|
||||
},
|
||||
timeout: this.config?.timeout || 10000
|
||||
});
|
||||
|
||||
const responseData = response.body as any;
|
||||
|
||||
if (!responseData?.chart?.result?.[0]) {
|
||||
throw new Error(`No data found for ticker ${request.ticker}`);
|
||||
}
|
||||
|
||||
const data = responseData.chart.result[0];
|
||||
const meta = data.meta;
|
||||
|
||||
if (!meta.regularMarketPrice) {
|
||||
throw new Error(`No price data available for ${request.ticker}`);
|
||||
}
|
||||
|
||||
const stockPrice: IStockPrice = {
|
||||
ticker: request.ticker.toUpperCase(),
|
||||
price: meta.regularMarketPrice,
|
||||
currency: meta.currency || 'USD',
|
||||
change: meta.regularMarketPrice - meta.previousClose,
|
||||
changePercent: ((meta.regularMarketPrice - meta.previousClose) / meta.previousClose) * 100,
|
||||
previousClose: meta.previousClose,
|
||||
timestamp: new Date(meta.regularMarketTime * 1000),
|
||||
provider: this.name,
|
||||
marketState: this.determineMarketState(meta),
|
||||
exchange: meta.exchange,
|
||||
exchangeName: meta.exchangeName
|
||||
};
|
||||
|
||||
return stockPrice;
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch price for ${request.ticker}:`, error);
|
||||
throw new Error(`Yahoo Finance: Failed to fetch price for ${request.ticker}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
public async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
|
||||
try {
|
||||
const symbols = request.tickers.join(',');
|
||||
const url = `${this.baseUrl}/v8/finance/spark?symbols=${symbols}&range=1d&interval=5m`;
|
||||
|
||||
const response = await plugins.smartrequest.getJson(url, {
|
||||
headers: {
|
||||
'User-Agent': this.userAgent
|
||||
},
|
||||
timeout: this.config?.timeout || 15000
|
||||
});
|
||||
|
||||
const responseData = response.body as any;
|
||||
const prices: IStockPrice[] = [];
|
||||
|
||||
for (const [ticker, data] of Object.entries(responseData)) {
|
||||
if (!data || typeof data !== 'object') continue;
|
||||
|
||||
const sparkData = data as any;
|
||||
if (!sparkData.previousClose || !sparkData.close?.length) {
|
||||
console.warn(`Incomplete data for ${ticker}, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const currentPrice = sparkData.close[sparkData.close.length - 1];
|
||||
const timestamp = sparkData.timestamp?.[sparkData.timestamp.length - 1];
|
||||
|
||||
prices.push({
|
||||
ticker: ticker.toUpperCase(),
|
||||
price: currentPrice,
|
||||
currency: sparkData.currency || 'USD',
|
||||
change: currentPrice - sparkData.previousClose,
|
||||
changePercent: ((currentPrice - sparkData.previousClose) / sparkData.previousClose) * 100,
|
||||
previousClose: sparkData.previousClose,
|
||||
timestamp: timestamp ? new Date(timestamp * 1000) : new Date(),
|
||||
provider: this.name,
|
||||
marketState: sparkData.marketState || 'REGULAR',
|
||||
exchange: sparkData.exchange,
|
||||
exchangeName: sparkData.exchangeName
|
||||
});
|
||||
}
|
||||
|
||||
if (prices.length === 0) {
|
||||
throw new Error('No valid price data received from batch request');
|
||||
}
|
||||
|
||||
return prices;
|
||||
} catch (error) {
|
||||
console.error(`Failed to fetch batch prices:`, error);
|
||||
throw new Error(`Yahoo Finance: Failed to fetch batch prices: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
public async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
// Test with a well-known ticker
|
||||
await this.fetchPrice({ ticker: 'AAPL' });
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn('Yahoo Finance provider is not available:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public supportsMarket(market: string): boolean {
|
||||
// Yahoo Finance supports most major markets
|
||||
const supportedMarkets = ['US', 'UK', 'DE', 'FR', 'JP', 'CN', 'HK', 'AU', 'CA'];
|
||||
return supportedMarkets.includes(market.toUpperCase());
|
||||
}
|
||||
|
||||
public supportsTicker(ticker: string): boolean {
|
||||
// Basic validation - Yahoo supports most tickers
|
||||
return /^[A-Z0-9\.\-]{1,10}$/.test(ticker.toUpperCase());
|
||||
}
|
||||
|
||||
private determineMarketState(meta: any): 'PRE' | 'REGULAR' | 'POST' | 'CLOSED' {
|
||||
const marketState = meta.marketState?.toUpperCase();
|
||||
|
||||
switch (marketState) {
|
||||
case 'PRE':
|
||||
return 'PRE';
|
||||
case 'POST':
|
||||
return 'POST';
|
||||
case 'REGULAR':
|
||||
return 'REGULAR';
|
||||
default:
|
||||
// Check if market is currently open based on timestamps
|
||||
const now = Date.now() / 1000;
|
||||
const regularMarketTime = meta.regularMarketTime;
|
||||
const timeDiff = now - regularMarketTime;
|
||||
|
||||
// If last update was more than 1 hour ago, market is likely closed
|
||||
return timeDiff > 3600 ? 'CLOSED' : 'REGULAR';
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user