Compare commits
6 Commits
Author | SHA1 | Date | |
---|---|---|---|
fea83153ba | |||
4716ef03ba | |||
3b76de0831 | |||
e94a6f8d5b | |||
1b1324d0f9 | |||
71a5a32198 |
1
.serena/.gitignore
vendored
Normal file
1
.serena/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
/cache
|
67
.serena/project.yml
Normal file
67
.serena/project.yml
Normal file
@@ -0,0 +1,67 @@
|
||||
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
|
||||
# * For C, use cpp
|
||||
# * For JavaScript, use typescript
|
||||
# Special requirements:
|
||||
# * csharp: Requires the presence of a .sln file in the project folder.
|
||||
language: typescript
|
||||
|
||||
# whether to use the project's gitignore file to ignore files
|
||||
# Added on 2025-04-07
|
||||
ignore_all_files_in_gitignore: true
|
||||
# list of additional paths to ignore
|
||||
# same syntax as gitignore, so you can use * and **
|
||||
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||
# Added (renamed) on 2025-04-07
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||
# Below is the complete list of tools for convenience.
|
||||
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||
# execute `uv run scripts/print_tool_overview.py`.
|
||||
#
|
||||
# * `activate_project`: Activates a project by name.
|
||||
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||
# * `delete_lines`: Deletes a range of lines within a file.
|
||||
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||
# * `execute_shell_command`: Executes a shell command.
|
||||
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||
# Should only be used in settings where the system prompt cannot be set,
|
||||
# e.g. in clients you have no control over, like Claude Desktop.
|
||||
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||
# * `read_file`: Reads a file within the project directory.
|
||||
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||
# * `remove_project`: Removes a project from the Serena configuration.
|
||||
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||
# * `switch_modes`: Activates modes by providing a list of their names
|
||||
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||
excluded_tools: []
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
project_name: "opendata"
|
27
changelog.md
27
changelog.md
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-10-11 - 1.7.0 - feat(stocks)
|
||||
Add Marketstack provider (EOD) with tests, exports and documentation updates
|
||||
|
||||
- Add MarketstackProvider implementation (ts/stocks/providers/provider.marketstack.ts) providing EOD single and batch fetching, availability checks and mapping to IStockPrice.
|
||||
- Export MarketstackProvider from ts/stocks/index.ts so it is available via the public API.
|
||||
- Add comprehensive Marketstack tests (test/test.marketstack.node.ts) covering registration, health checks, single/batch fetches, caching, ticker/market validation, provider stats and sample output.
|
||||
- Update README with Marketstack usage examples, configuration, API key instructions and provider/health documentation.
|
||||
- Bump dev dependency @git.zone/tstest to ^2.4.2 in package.json.
|
||||
- Add project helper/config files (.claude/settings.local.json, .serena/project.yml and .serena/.gitignore) to support CI/tooling.
|
||||
|
||||
## 2025-09-24 - 1.6.1 - fix(stocks)
|
||||
Fix Yahoo provider request handling and bump dependency versions
|
||||
|
||||
- Refactored Yahoo Finance provider to use SmartRequest.create() builder and await response.json() for HTTP responses (replaces direct getJson usage).
|
||||
- Improved batch and single-price fetching to use the SmartRequest API, keeping User-Agent header and timeouts.
|
||||
- Added a compile-time type-check alias to ensure IStockPrice matches tsclass.finance.IStockPrice.
|
||||
- Bumped development and runtime dependency versions (notable bumps include @git.zone/tsbuild, @git.zone/tstest, @push.rocks/qenv, @push.rocks/smartarchive, @push.rocks/smartdata, @push.rocks/smartfile, @push.rocks/smartlog, @push.rocks/smartpath, @push.rocks/smartrequest, @tsclass/tsclass).
|
||||
- Added .claude/settings.local.json to grant local CI permissions for a few Bash commands.
|
||||
|
||||
## 2025-07-12 - 1.6.0 - feat(readme)
|
||||
Revamp documentation and package description for enhanced clarity
|
||||
|
||||
- Restructured README to highlight real-time stock data and German business data, streamlining quick start and advanced examples
|
||||
- Updated package.json description to better reflect library capabilities
|
||||
- Added .claude/settings.local.json to define permissions for external tools
|
||||
- Refined code examples in tests and documentation for improved clarity and consistency
|
||||
|
||||
## 2025-04-09 - 1.5.3 - fix(test)
|
||||
Await file writes in Handelsregister tests to ensure all downloads complete before test end
|
||||
|
||||
|
24
package.json
24
package.json
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"name": "@fin.cx/opendata",
|
||||
"version": "1.5.4",
|
||||
"version": "1.7.0",
|
||||
"private": false,
|
||||
"description": "A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.",
|
||||
"description": "A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
@@ -14,29 +14,29 @@
|
||||
"buildDocs": "(tsdoc)"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.6.4",
|
||||
"@git.zone/tsbuild": "^2.6.8",
|
||||
"@git.zone/tsbundle": "^2.5.1",
|
||||
"@git.zone/tsrun": "^1.3.3",
|
||||
"@git.zone/tstest": "^2.3.1",
|
||||
"@git.zone/tstest": "^2.4.2",
|
||||
"@types/node": "^22.14.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/lik": "^6.2.2",
|
||||
"@push.rocks/qenv": "^6.1.0",
|
||||
"@push.rocks/smartarchive": "^4.0.39",
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@push.rocks/smartarchive": "^4.2.2",
|
||||
"@push.rocks/smartarray": "^1.1.0",
|
||||
"@push.rocks/smartbrowser": "^2.0.8",
|
||||
"@push.rocks/smartdata": "^5.15.1",
|
||||
"@push.rocks/smartdata": "^5.16.4",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.2.5",
|
||||
"@push.rocks/smartlog": "^3.1.8",
|
||||
"@push.rocks/smartpath": "^5.0.18",
|
||||
"@push.rocks/smartfile": "^11.2.7",
|
||||
"@push.rocks/smartlog": "^3.1.10",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartpromise": "^4.2.3",
|
||||
"@push.rocks/smartrequest": "^2.1.0",
|
||||
"@push.rocks/smartrequest": "^4.3.1",
|
||||
"@push.rocks/smartstream": "^3.2.5",
|
||||
"@push.rocks/smartunique": "^3.0.9",
|
||||
"@push.rocks/smartxml": "^1.1.1",
|
||||
"@tsclass/tsclass": "^9.2.0"
|
||||
"@tsclass/tsclass": "^9.3.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
4156
pnpm-lock.yaml
generated
4156
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
935
readme.md
935
readme.md
@@ -1,649 +1,360 @@
|
||||
# @fin.cx/opendata
|
||||
A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.
|
||||
|
||||
## Install
|
||||
🚀 **Real-time financial data and German business intelligence in one powerful TypeScript library**
|
||||
|
||||
To install the @fin.cx/opendata package, you can use npm or yarn as your package manager. The installation process is simple and straightforward.
|
||||
Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API.
|
||||
|
||||
Using npm:
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install @fin.cx/opendata
|
||||
|
||||
Using yarn:
|
||||
|
||||
yarn add @fin.cx/opendata
|
||||
|
||||
## Usage
|
||||
|
||||
The @fin.cx/opendata library is a versatile tool that empowers developers to integrate comprehensive open business data into their systems. This library is particularly tailored for German companies, offering functionalities that include creating, retrieving, updating, and deleting business records as well as processing large volumes of JSONL data from external sources. In addition to core database operations via MongoDB, the library provides integration with web-based services, primarily through a hands-on Handelsregister processor that utilizes browser automation for searching and downloading documents.
|
||||
|
||||
In this section, we will extensively detail multiple usage scenarios, ensuring that every feature the module offers is thoroughly explored. All examples in this documentation employ ECMAScript Module (ESM) syntax and TypeScript, highlighting proper asynchronous handling, error management, and advanced integration with other dependencies. We will walk you through environment setup, initializing the package, managing business records, processing bulk JSONL data, interacting with the Handelsregister for on-demand document retrieval, and much more. Each example is constructed to expose every nuance of the module's behavior and usage.
|
||||
|
||||
For clarity, we will split this section into multiple parts:
|
||||
|
||||
1. Environment Setup and Initializing the Library
|
||||
2. Managing Business Records (CRUD Operations)
|
||||
3. Bulk Data Processing and Importing via JSONL Streams
|
||||
4. Integrating with the Handelsregister: Detailed Demonstrations
|
||||
5. Advanced Examples: Combined Operations and Edge Cases
|
||||
6. Error Handling and Data Validation
|
||||
7. Testing and Automated Workflows
|
||||
|
||||
Throughout these examples, we will examine how each class and method interacts with the underlying MongoDB database and the system's file structure. We assume you have a running MongoDB instance and that your environment is configured with the necessary variables.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 1. Environment Setup and Initializing the Library
|
||||
|
||||
Before diving into any operations, ensure that your development environment is properly configured. The @fin.cx/opendata library mandates several environment variables for connecting to your MongoDB instance. For a smooth experience, it is advisable to use a .env file or any secure secrets management tool that suits your workflow. The required environment variables include:
|
||||
|
||||
• MONGODB_URL – The connection string URL for your MongoDB instance.
|
||||
• MONGODB_NAME – The name of the database that the module will interact with.
|
||||
• MONGODB_USER – MongoDB username required for authentication.
|
||||
• MONGODB_PASS – The password for the MongoDB user.
|
||||
|
||||
Below is an example .env file for local development:
|
||||
|
||||
```
|
||||
MONGODB_URL=mongodb://localhost:27017
|
||||
MONGODB_NAME=opendataDB
|
||||
MONGODB_USER=myUser
|
||||
MONGODB_PASS=myPass
|
||||
# or
|
||||
pnpm add @fin.cx/opendata
|
||||
```
|
||||
|
||||
Once these variables are set, the library can fetch them using the integrated qenv tool. The following code snippet demonstrates how to import and initialize the library:
|
||||
## Quick Start
|
||||
|
||||
### 📈 Stock Market Data
|
||||
|
||||
Get market data with EOD (End-of-Day) pricing:
|
||||
|
||||
```typescript
|
||||
import { StockPriceService, MarketstackProvider } from '@fin.cx/opendata';
|
||||
|
||||
// Initialize the service with caching
|
||||
const stockService = new StockPriceService({
|
||||
ttl: 60000, // Cache for 1 minute
|
||||
maxEntries: 1000 // Max cached symbols
|
||||
});
|
||||
|
||||
// Register Marketstack provider with API key
|
||||
stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
|
||||
priority: 100,
|
||||
retryAttempts: 3
|
||||
});
|
||||
|
||||
// Get single stock price
|
||||
const apple = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
console.log(`Apple: $${apple.price} (${apple.changePercent.toFixed(2)}%)`);
|
||||
|
||||
// Get multiple prices at once (batch fetching)
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA']
|
||||
});
|
||||
|
||||
// 125,000+ tickers across 72+ exchanges worldwide
|
||||
const internationalStocks = await stockService.getPrices({
|
||||
tickers: ['AAPL', 'VOD.LON', 'SAP.DEX', 'TM', 'BABA']
|
||||
});
|
||||
```
|
||||
|
||||
### 🏢 German Business Data
|
||||
|
||||
Access comprehensive data on German companies:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
const startOpenDataInstance = async () => {
|
||||
const openData = new OpenData();
|
||||
|
||||
try {
|
||||
console.log('Starting OpenData instance...');
|
||||
await openData.start();
|
||||
console.log('OpenData instance started successfully.');
|
||||
|
||||
// Invoke sample operations:
|
||||
await demonstrateBusinessRecordsOperations(openData);
|
||||
await demonstrateBulkDataProcessing(openData);
|
||||
await demonstrateHandelsregisterOperations(openData);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during initialization:', error);
|
||||
} finally {
|
||||
console.log('Stopping OpenData instance...');
|
||||
await openData.stop();
|
||||
console.log('OpenData instance stopped.');
|
||||
}
|
||||
};
|
||||
|
||||
startOpenDataInstance();
|
||||
```
|
||||
|
||||
In this snippet, we import the OpenData class from the module and execute its start and stop routines to ensure that the MongoDB connection is properly initialized and terminated. Notice that we move on to different demonstration functions that showcase individual features.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 2. Managing Business Records (CRUD Operations)
|
||||
|
||||
Central to the @fin.cx/opendata library is the management of business records. The BusinessRecord class encapsulates data pertaining to companies, allowing you to create new records, retrieve existing ones, update information, and delete entries when necessary. The following examples illustrate each operation within a robust context.
|
||||
|
||||
#### a) Creating a Business Record
|
||||
|
||||
Creating a new business record in the openData instance is straightforward. You instantiate a new record and populate its data properties with relevant details such as company name, address, registration number, managing directors, and much more. The sample below uses the embedded CBusinessRecord manager to generate a new record:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const createBusinessRecordExample = async (openData: OpenData) => {
|
||||
const businessRecord = new openData.CBusinessRecord();
|
||||
|
||||
businessRecord.data = {
|
||||
name: "Innovative Solutions GmbH",
|
||||
address: "Musterstraße 1",
|
||||
postalCode: "10115",
|
||||
// Create a business record
|
||||
const company = new openData.CBusinessRecord();
|
||||
company.data = {
|
||||
name: "TechStart GmbH",
|
||||
city: "Berlin",
|
||||
country: "Germany",
|
||||
phone: "+49 30 123456",
|
||||
email: "contact@innovativesolutions.de",
|
||||
website: "https://innovativesolutions.de",
|
||||
businessType: "GmbH",
|
||||
registrationId: "District court Berlin HRB 987654",
|
||||
legalForm: "GmbH",
|
||||
managingDirectors: ["Max Mustermann", "Erika Musterfrau"],
|
||||
foundingDate: new Date("2018-05-10").toISOString(),
|
||||
capital: "250,000 EUR",
|
||||
purpose: "Technology development and consulting services",
|
||||
lastUpdate: new Date().toISOString()
|
||||
registrationId: "HRB 123456",
|
||||
// ... more fields
|
||||
};
|
||||
await company.save();
|
||||
|
||||
try {
|
||||
await businessRecord.save();
|
||||
console.log('BusinessRecord created successfully:', businessRecord);
|
||||
} catch (error) {
|
||||
console.error('Error creating business record:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
In this example, after setting the business record fields, the record is saved to the MongoDB collection using the save method. The system ensures that the newly created record receives a unique identifier by generating a new ID when saving the document.
|
||||
|
||||
#### b) Retrieving Business Records
|
||||
|
||||
To retrieve business records, you can search by various fields such as city, business name, or registration details. The system utilizes MongoDB queries to filter and return relevant documents. Below is a sample function that retrieves all records for companies based in a particular city:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
import type { BusinessRecord } from '@fin.cx/opendata';
|
||||
|
||||
export const retrieveRecordsByCity = async (openData: OpenData, city: string) => {
|
||||
try {
|
||||
const records = await openData.db
|
||||
.collection<BusinessRecord>('businessrecords')
|
||||
.find({ city })
|
||||
.toArray();
|
||||
|
||||
console.log(`Retrieved ${records.length} records for city ${city}.`);
|
||||
console.log(records);
|
||||
return records;
|
||||
} catch (error) {
|
||||
console.error('Error retrieving business records:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
This method queries the "businessrecords" collection using a simple filter and converts the cursor into an array of records. You can extend the query to filter by more sophisticated criteria as needed.
|
||||
|
||||
#### c) Updating an Existing Business Record
|
||||
|
||||
Modifying the details of an exisiting record is a common operation. First, you need to retrieve the record from the database. Once the record is loaded, you can make changes to its properties and then save the updated record back to the database. The following example demonstrates this with a change to the company’s phone number and last update timestamp:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const updateBusinessRecordExample = async (openData: OpenData, recordId: string) => {
|
||||
try {
|
||||
// Retrieve the record by its id using the manager’s helper
|
||||
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
|
||||
if (!businessRecord) {
|
||||
console.log(`No business record found with id: ${recordId}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update some fields
|
||||
businessRecord.data.phone = "+49 30 654321";
|
||||
businessRecord.data.lastUpdate = new Date().toISOString();
|
||||
|
||||
// Save the updated record into the database
|
||||
await businessRecord.save();
|
||||
console.log("Business record updated successfully:", businessRecord);
|
||||
} catch (error) {
|
||||
console.error('Error updating business record:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
This code snippet presents a robust pattern where errors are caught and logged, ensuring that any update issues can be diagnosed easily.
|
||||
|
||||
#### d) Deleting a Business Record
|
||||
|
||||
The deletion of a record is as vital as its creation and modification. The library provides a delete method that removes the specified record from the database. Below is a simple function to delete a business record by its identifier:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const deleteBusinessRecordExample = async (openData: OpenData, recordId: string) => {
|
||||
try {
|
||||
const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
|
||||
if (businessRecord) {
|
||||
await businessRecord.delete();
|
||||
console.log(`Successfully deleted business record with id: ${recordId}`);
|
||||
} else {
|
||||
console.log(`No business record found with id: ${recordId}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error deleting business record:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
Through this example, you can integrate safe deletion practices in your application, removing outdated or incorrect records without compromising database integrity.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 3. Bulk Data Processing and Importing via JSONL Streams
|
||||
|
||||
One of the powerful features of the @fin.cx/opendata module is its ability to process large datasets provided in the JSON Lines (JSONL) format. The JsonlDataProcessor class is designed to handle streaming data, processing each record concurrently, and efficiently updating the database.
|
||||
|
||||
This bulk data ingestion mechanism is particularly useful when dealing with large-scale datasets such as the German companies' open data that the module fetches from official data portals. The process involves decompressing, streaming, and parsing data by leveraging pipelines of smart streams and concurrent processors.
|
||||
|
||||
Below is an extended example demonstrating how to process a JSONL data file from a given URL:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
// This function demonstrates bulk data processing using the JSONL data processor.
|
||||
// The dataUrl parameter is optional and, if not provided, defaults to the official open data URL.
|
||||
export const demonstrateBulkDataProcessing = async (openData: OpenData, dataUrl?: string) => {
|
||||
try {
|
||||
console.log('Starting bulk data processing...');
|
||||
await openData.jsonLDataProcessor.processDataFromUrl(dataUrl);
|
||||
console.log('Bulk data processing completed successfully.');
|
||||
} catch (error) {
|
||||
console.error('Error during bulk data processing:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
In the processDataFromUrl implementation, the library uses a pipeline of smart streams. After downloading the compressed file, it decompresses it and splits the content into discrete JSON lines. The processor then concurrently applies a handler function to each JSON entry. This function extracts relevant company details, instantiates a new BusinessRecord, associates parsed data (for example, registration attributes from German registers), and saves the record to MongoDB.
|
||||
|
||||
A deeper dive into the processing mechanism:
|
||||
• The JSONL data is received as a binary (Buffer) stream.
|
||||
• The stream is piped into a duplex stream that splits the text by newline characters.
|
||||
• Each line is parsed into a JSON object and passed into an asynchronous processing function.
|
||||
• This function creates a new business record and sets properties such as the company name and its registration details, derived from the JSON entry.
|
||||
• As the processor moves through the stream, it logs progress every 10,000 records to give feedback on its bulk processing status.
|
||||
|
||||
By supporting concurrency (with a configurable concurrency limit, e.g., 1000 simultaneous operations), the library ensures that even gigabytes of data are processed efficiently without hitting memory bottlenecks.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 4. Integrating with the Handelsregister: Detailed Demonstrations
|
||||
|
||||
In addition to CRUD operations and bulk processing, the module includes an integrated Handelsregister system. This sophisticated component leverages a headless browser (via the smartbrowser instance) to interact with the official Handelsregister website. Through this integration, you can search for companies, navigate to specific pages, trigger file downloads (such as PDF or XML data), and parse the downloaded content for further processing.
|
||||
|
||||
#### a) Starting the Handelsregister
|
||||
|
||||
Before executing any search or download operations, the Handelsregister system must be started. The start method initializes required resources including starting a headless browser, ensuring download directories are created, and preparing asynchronous stacks for exclusive execution.
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const demonstrateHandelsregisterStart = async (openData: OpenData) => {
|
||||
try {
|
||||
console.log('Starting Handelsregister services...');
|
||||
await openData.handelsregister.start();
|
||||
console.log('Handelsregister ready.');
|
||||
} catch (error) {
|
||||
console.error('Error starting Handelsregister service:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
#### b) Searching for a Company Using the Handelsregister
|
||||
|
||||
A common use case is to search for a company by its name. The Handelsregister system creates a dedicated browser page, enters the search criteria into the input fields, selects the appropriate options (such as radio buttons for search type), and clicks the “Find” button. The following function demonstrates how to incorporate these actions:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const searchCompanyExample = async (openData: OpenData, companyName: string, limit: number = 100) => {
|
||||
try {
|
||||
console.log(`Searching for company with name "${companyName}"...`);
|
||||
const records = await openData.handelsregister.searchCompany(companyName, limit);
|
||||
console.log(`Found ${records.length} matching records for "${companyName}".`);
|
||||
console.log('Records:', records);
|
||||
return records;
|
||||
} catch (error) {
|
||||
console.error('Error searching for company:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
In this scenario, the Handelsregister component uses internal helper functions:
|
||||
• getNewPage – to create a new browser page with file download behavior enabled.
|
||||
• navigateToPage – which navigates to the “Normal search” page.
|
||||
• Input events – to simulate typing in search criteria.
|
||||
• UI interactions – to select options and trigger the search.
|
||||
|
||||
#### c) Retrieving Detailed Data and Triggering Downloads
|
||||
|
||||
After obtaining general search results, you may wish to retrieve more detailed information about a specific company. Provided you have the parsed registration data (which typically includes the registration court, type, and number), you can instruct the system to navigate to a detailed view and trigger file downloads. These files might include the company’s official registry entry (as an XML file) and additional documents (such as a PDF summary).
|
||||
|
||||
The example below details how to use the Handelsregister functionality to focus on a specific company by leveraging its registration details, then download both SI and AD files:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const getDetailedCompanyData = async (openData: OpenData, registrationData: { court?: string; type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR'; number?: string; }) => {
|
||||
try {
|
||||
console.log('Retrieving detailed company data...');
|
||||
const result = await openData.handelsregister.getSpecificCompany(registrationData);
|
||||
console.log('Retrieved detailed company data.');
|
||||
console.log('Business Records:', result.businessRecords);
|
||||
console.log('Downloaded Files:', result.files);
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error('Error retrieving detailed company data:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
In the above example:
|
||||
• The getSpecificCompany method triggers navigation through various UI elements:
|
||||
– Selecting the register type via a dropdown.
|
||||
– Inputting the register number.
|
||||
– Choosing the appropriate register court.
|
||||
• Then, after clicking the “Find” button, the system waits for the results, verifies the visual components on the page, and initiates file downloads.
|
||||
• The downloaded files are renamed according to their type (SI for XML and AD for PDF) and are returned for further processing or storage.
|
||||
|
||||
#### d) Downloading and Processing Files
|
||||
|
||||
The Handelsregister component not only triggers file downloads but also includes utility functions that wait for downloads to complete, clear temporary directories, and output the file objects. You may want to use these file objects to persist data locally, parse file content, or send the data downstream for further analysis.
|
||||
|
||||
Below is an example that covers downloading and saving the files into a custom directory for post-download analysis:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
import * as path from 'path';
|
||||
|
||||
export const downloadAndSaveFilesExample = async (openData: OpenData, registrationData: { court?: string; type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR'; number?: string; }) => {
|
||||
try {
|
||||
console.log('Initiating specific company download...');
|
||||
const result = await openData.handelsregister.getSpecificCompany(registrationData);
|
||||
const saveDirectory = path.join(process.cwd(), 'downloaded_files');
|
||||
|
||||
// Save each downloaded file to the specified directory
|
||||
for (const file of result.files) {
|
||||
await file.writeToDir(saveDirectory);
|
||||
console.log(`File saved: ${file.path}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during file download and save process:', error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
This function demonstrates a complete flow from launching the Handelsregister detailed company search to saving the downloaded files to disk. This example is particularly useful in scenarios where the downloaded documents need to be processed further, such as converting XML to JSON or extracting text from PDFs.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 5. Advanced Examples: Combined Operations and Edge Cases
|
||||
|
||||
Given the numerous functionalities offered by the library, you can combine various operations to create more complex workflows. One such example is an end-to-end pipeline that:
|
||||
1. Initializes the open data instance.
|
||||
2. Processes an initial bulk data import.
|
||||
3. Searches for key business records that match specific criteria.
|
||||
4. Updates individual records based on additional data retrieved from the Handelsregister.
|
||||
5. Handles error conditions and retries processes where necessary.
|
||||
|
||||
The following advanced example integrates these steps:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
const advancedWorkflowExample = async () => {
|
||||
const openData = new OpenData();
|
||||
|
||||
try {
|
||||
console.log('Starting advanced workflow...');
|
||||
await openData.start();
|
||||
|
||||
// Step 1: Bulk data import from external JSONL source
|
||||
console.log('Building initial database from bulk import...');
|
||||
await openData.buildInitialDb();
|
||||
|
||||
// Step 2: Search for companies in a selected area (for instance, Munich)
|
||||
console.log('Retrieving companies located in Munich...');
|
||||
const munichRecords = await openData.db
|
||||
// Search companies by city
|
||||
const berlinCompanies = await openData.db
|
||||
.collection('businessrecords')
|
||||
.find({ city: "Munich" })
|
||||
.find({ city: "Berlin" })
|
||||
.toArray();
|
||||
console.log(`Found ${munichRecords.length} companies in Munich.`);
|
||||
|
||||
// Step 3: For each record, perform an update operation based on new file downloads
|
||||
for (const record of munichRecords) {
|
||||
try {
|
||||
console.log(`Updating record for company: ${record.data.name}`);
|
||||
// Assuming the record contains parsed registration info
|
||||
if (record.data.germanParsedRegistration) {
|
||||
const detailedData = await openData.handelsregister.getSpecificCompany(record.data.germanParsedRegistration);
|
||||
// Update business record with new information (e.g., registration files or updated details)
|
||||
record.data.lastUpdate = new Date().toISOString();
|
||||
// You might want to add additional fields based on the downloaded file data
|
||||
await record.save();
|
||||
console.log(`Updated record for ${record.data.name}.`);
|
||||
} else {
|
||||
console.log(`No registration data available for ${record.data.name}; skipping update.`);
|
||||
}
|
||||
} catch (innerError) {
|
||||
console.error(`Error updating record for ${record.data.name}:`, innerError);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Demonstrate retrieval and deletion
|
||||
const recordToDeleteId = munichRecords.length > 0 ? munichRecords[0].id : null;
|
||||
if (recordToDeleteId) {
|
||||
console.log(`Deleting record with id: ${recordToDeleteId}`);
|
||||
const recordToDelete = await openData.CBusinessRecord.getInstance(recordToDeleteId);
|
||||
if (recordToDelete) {
|
||||
await recordToDelete.delete();
|
||||
console.log(`Record ${recordToDeleteId} deleted successfully.`);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Advanced workflow encountered an error:', error);
|
||||
} finally {
|
||||
console.log('Ending advanced workflow and stopping OpenData instance.');
|
||||
await openData.stop();
|
||||
}
|
||||
};
|
||||
|
||||
advancedWorkflowExample();
|
||||
```
|
||||
|
||||
This advanced workflow not only illustrates the coordinated use of bulk data import, search, update, and delete operations but also demonstrates the integration of browser automation for fetching detailed data. The error handling at each step ensures that even if a particular operation fails, the workflow continues in a controlled fashion.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 6. Error Handling and Data Validation
|
||||
|
||||
Robust systems must gracefully handle errors and ensure data consistency. The @fin.cx/opendata library has built-in error handling for asynchronous operations, whether connecting to MongoDB, processing JSON streams, or interacting with web pages. In addition, each BusinessRecord instance provides a validate method that performs basic checks (for instance, ensuring that a company name is present) before a record is saved into the database.
|
||||
|
||||
The snippet below shows how to wrap operations in try/catch blocks and use the validate method:
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
export const validateAndSaveRecord = async (openData: OpenData) => {
|
||||
const record = new openData.CBusinessRecord();
|
||||
record.data = {
|
||||
name: "Validation Test Corp",
|
||||
address: "Teststraße 99",
|
||||
postalCode: "12345",
|
||||
city: "Teststadt",
|
||||
country: "Germany",
|
||||
phone: "+49 123 456789",
|
||||
email: "test@testcorp.de",
|
||||
businessType: "AG",
|
||||
registrationId: "District court Teststadt HRB 111111",
|
||||
legalForm: "AG",
|
||||
managingDirectors: ["Test Director"],
|
||||
foundingDate: new Date().toISOString(),
|
||||
capital: "1,000,000 EUR",
|
||||
purpose: "Testing for data validation",
|
||||
lastUpdate: new Date().toISOString()
|
||||
};
|
||||
|
||||
try {
|
||||
// Validate record data before saving.
|
||||
await record.validate();
|
||||
await record.save();
|
||||
console.log("Record validated and saved successfully.");
|
||||
} catch (error) {
|
||||
console.error("Error validating or saving record:", error);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
Using proper error handling ensures that the entire system remains reliable, and any data validation issues are caught early during development or in production.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### 7. Testing and Automated Workflows
|
||||
|
||||
To support continuous integration and adherence to best practices, the @fin.cx/opendata module includes tests written with @push.rocks/tapbundle. You should consider incorporating these tests in your development workflow. The tests verify all main functionalities including instance initialization, bulk data import, Handelsregister operations, and CRUD operations for BusinessRecords.
|
||||
|
||||
Below is an example of a simple test written in TypeScript using ESM that makes use of the module:
|
||||
|
||||
```typescript
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
let testOpenDataInstance: OpenData;
|
||||
|
||||
tap.test('Instance creation', async () => {
|
||||
testOpenDataInstance = new OpenData();
|
||||
expect(testOpenDataInstance).toBeInstanceOf(OpenData);
|
||||
});
|
||||
|
||||
tap.test('Start instance', async () => {
|
||||
await testOpenDataInstance.start();
|
||||
});
|
||||
|
||||
tap.test('Perform bulk import', async () => {
|
||||
await testOpenDataInstance.buildInitialDb();
|
||||
});
|
||||
|
||||
tap.test('Stop instance', async () => {
|
||||
await testOpenDataInstance.stop();
|
||||
});
|
||||
|
||||
tap.start();
|
||||
```
|
||||
|
||||
This test code is designed to verify that the OpenData instance is successfully created, started, performs the critical bulk import operation, and is properly shutdown. Integration tests for the Handelsregister functionality follow a similar pattern and ensure that the browser automation routines and file download processes complete without errors.
|
||||
|
||||
────────────────────────────────────────────
|
||||
### Comprehensive Example: Full Cycle from Initialization to Cleanup
|
||||
|
||||
To better illustrate how one might combine several aspects of the module in a production scenario, here's a comprehensive example that ties together initialization, CRUD operations, bulk processing, and Handelsregister interactions. This full-cycle example is written in TypeScript using ESM syntax and demonstrates how to build a production-grade data update and management pipeline.
|
||||
|
||||
```typescript
|
||||
import { OpenData } from '@fin.cx/opendata';
|
||||
|
||||
const runFullCyclePipeline = async () => {
|
||||
const openData = new OpenData();
|
||||
|
||||
try {
|
||||
// Initialize the module and connect to MongoDB
|
||||
console.log('Initializing the OpenData module...');
|
||||
await openData.start();
|
||||
|
||||
// Step 1: Bulk Import - Build the initial database from downloaded open data
|
||||
console.log('Starting bulk data import from JSONL source...');
|
||||
// Import bulk data from official sources
|
||||
await openData.buildInitialDb();
|
||||
|
||||
// Step 2: Business Record Management - Create a sample business record
|
||||
console.log('Creating a new business record...');
|
||||
const sampleRecord = new openData.CBusinessRecord();
|
||||
sampleRecord.data = {
|
||||
name: "Sample Enterprise GmbH",
|
||||
address: "Innovation Avenue 101",
|
||||
postalCode: "80807",
|
||||
city: "Munich",
|
||||
country: "Germany",
|
||||
phone: "+49 89 111222",
|
||||
email: "info@sampleenterprise.de",
|
||||
website: "https://sampleenterprise.de",
|
||||
businessType: "GmbH",
|
||||
registrationId: "District court Munich HRB 555555",
|
||||
legalForm: "GmbH",
|
||||
managingDirectors: ["Director A", "Director B"],
|
||||
foundingDate: new Date("2015-06-15").toISOString(),
|
||||
capital: "500,000 EUR",
|
||||
purpose: "Holistic business solutions and data processing",
|
||||
lastUpdate: new Date().toISOString()
|
||||
};
|
||||
|
||||
await sampleRecord.save();
|
||||
console.log('Sample business record created with id:', sampleRecord.id);
|
||||
|
||||
// Step 3: Retrieve business records for a specific location
|
||||
console.log('Retrieving business records for Munich...');
|
||||
const munichRecords = await openData.db
|
||||
.collection('businessrecords')
|
||||
.find({ city: "Munich" })
|
||||
.toArray();
|
||||
console.log(`Found ${munichRecords.length} records for Munich.`);
|
||||
|
||||
// Step 4: Update an existing record
|
||||
if (munichRecords.length > 0) {
|
||||
const recordToUpdateId = munichRecords[0].id;
|
||||
console.log(`Updating business record with id: ${recordToUpdateId}`);
|
||||
const recordToUpdate = await openData.CBusinessRecord.getInstance(recordToUpdateId);
|
||||
if (recordToUpdate) {
|
||||
recordToUpdate.data.phone = "+49 89 999888";
|
||||
recordToUpdate.data.lastUpdate = new Date().toISOString();
|
||||
await recordToUpdate.save();
|
||||
console.log('Business record updated:', recordToUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Use Handelsregister to search for a specific company
|
||||
console.log('Using Handelsregister to search for a specific company...');
|
||||
const searchResults = await openData.handelsregister.searchCompany("Step Beyond GmbH", 20);
|
||||
if (searchResults && searchResults.length > 0) {
|
||||
const registrationData = searchResults[0].germanParsedRegistration;
|
||||
console.log('Retrieved registration data:', registrationData);
|
||||
|
||||
// Step 6: Retrieve detailed info and download files for the specific company
|
||||
console.log('Fetching detailed data for the identified company...');
|
||||
const detailedInfo = await openData.handelsregister.getSpecificCompany(registrationData);
|
||||
console.log('Detailed company data received:', detailedInfo);
|
||||
|
||||
// Optionally, save the downloaded files to a designated directory
|
||||
for (const downloadedFile of detailedInfo.files) {
|
||||
await downloadedFile.writeToDir('./output_files');
|
||||
console.log(`Downloaded file saved at: ${downloadedFile.path}`);
|
||||
}
|
||||
} else {
|
||||
console.log('No matching records found for detailed company data retrieval.');
|
||||
}
|
||||
|
||||
// Step 7: Validate and save a record to demonstrate error handling and validation
|
||||
console.log('Validating and saving a new test record...');
|
||||
await validateAndSaveRecord(openData);
|
||||
|
||||
} catch (error) {
|
||||
console.error('An error occurred during the full cycle pipeline operation:', error);
|
||||
} finally {
|
||||
// Final cleanup: Stop the OpenData module and release all resources
|
||||
console.log('Finalizing: stopping the OpenData module...');
|
||||
await openData.stop();
|
||||
console.log('Pipeline completed and all resources have been cleaned up.');
|
||||
}
|
||||
};
|
||||
|
||||
runFullCyclePipeline();
|
||||
```
|
||||
|
||||
In this example, the entire processing cycle is constructed to mimic a realistic scenario. The pipeline:
|
||||
• Starts by connecting to your database.
|
||||
• Imports extensive JSONL open data.
|
||||
• Creates, retrieves, updates, and deletes business records.
|
||||
• Interacts with the Handelsregister for advanced company-specific operations.
|
||||
• Implements robust error handling and validation routines, ensuring that each step is verifiable.
|
||||
• Finally, ensures that resources such as MongoDB connections and headless browser sessions are responsibly closed.
|
||||
## Features
|
||||
|
||||
────────────────────────────────────────────
|
||||
### Final Thoughts on Module Integration
|
||||
### 🎯 Stock Market Module
|
||||
|
||||
The @fin.cx/opendata library is designed to cater to a wide range of business data management needs. Whether you are an enterprise looking to integrate updated open data for decision-making or a developer looking to build data-rich applications with a focus on German companies, this library provides the tools and abstractions necessary to build robust solutions.
|
||||
- **Marketstack API** - End-of-Day (EOD) data for 125,000+ tickers across 72+ exchanges
|
||||
- **Stock prices** for stocks, ETFs, indices, and more
|
||||
- **Batch operations** - fetch 100+ symbols in one request
|
||||
- **Smart caching** - configurable TTL, automatic invalidation
|
||||
- **Extensible provider system** - easily add new data sources
|
||||
- **Retry logic** - configurable retry attempts and delays
|
||||
- **Type-safe** - full TypeScript support with detailed interfaces
|
||||
|
||||
Every component—from the smart data management for business records to the advanced streaming and concurrent processing of JSONL files—is built with scalability and ease of use in mind. Integration with the Handelsregister via browser automation further extends its reach, providing dynamic access to official data sources in real-time.
|
||||
### 🇩🇪 German Business Intelligence
|
||||
|
||||
As demonstrated in the examples above, each sub-component of the library is independent yet harmoniously integrated into a cohesive user experience. The use of ESM syntax throughout the module and the strict adherence to TypeScript definitions enhances reliability, maintainability, and the overall developer experience.
|
||||
- **MongoDB integration** for scalable data storage
|
||||
- **Bulk JSONL import** from official German data sources
|
||||
- **Handelsregister automation** - automated document retrieval
|
||||
- **CRUD operations** with validation
|
||||
- **Streaming processing** for multi-GB datasets
|
||||
|
||||
By following the usage scenarios provided in this documentation, you should now have a deep understanding of how to:
|
||||
• Set up your environment and initialize the OpenData instance.
|
||||
• Perform CRUD operations on business records.
|
||||
• Efficiently process thousands of records from external JSONL sources.
|
||||
• Integrate and automate Handelsregister interactions for detailed company data retrieval.
|
||||
• Combine all building blocks into advanced automated workflows that support large-scale enterprise applications.
|
||||
## Advanced Examples
|
||||
|
||||
Feel free to explore, extend, and customize these examples to suit your project’s unique requirements. The library is designed with extensibility in mind, and additional utility functions or integrations can be added based on your needs.
|
||||
### Market Dashboard
|
||||
|
||||
We encourage you to integrate these practices into your development processes, run the provided tests, and contribute to further enhancements that can benefit the entire community of developers working on open data management systems.
|
||||
Create an EOD market overview:
|
||||
|
||||
Happy coding and data integrating!
|
||||
```typescript
|
||||
const indicators = [
|
||||
// Indices
|
||||
{ ticker: '^GSPC', name: 'S&P 500' },
|
||||
{ ticker: '^DJI', name: 'DOW Jones' },
|
||||
|
||||
// Tech Giants
|
||||
{ ticker: 'AAPL', name: 'Apple' },
|
||||
{ ticker: 'MSFT', name: 'Microsoft' },
|
||||
{ ticker: 'GOOGL', name: 'Alphabet' },
|
||||
{ ticker: 'AMZN', name: 'Amazon' },
|
||||
{ ticker: 'TSLA', name: 'Tesla' }
|
||||
];
|
||||
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: indicators.map(i => i.ticker)
|
||||
});
|
||||
|
||||
// Display with color-coded changes
|
||||
prices.forEach(price => {
|
||||
const indicator = indicators.find(i => i.ticker === price.ticker);
|
||||
const arrow = price.change >= 0 ? '↑' : '↓';
|
||||
const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
|
||||
|
||||
console.log(
|
||||
`${indicator.name.padEnd(15)} ${price.price.toFixed(2).padStart(10)} ` +
|
||||
`${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m`
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
### Provider Health and Statistics
|
||||
|
||||
Monitor your provider health and track usage:
|
||||
|
||||
```typescript
|
||||
// Check provider health
|
||||
const health = await stockService.checkProvidersHealth();
|
||||
console.log(`Marketstack: ${health.get('Marketstack') ? '✅' : '❌'}`);
|
||||
|
||||
// Get provider statistics
|
||||
const stats = stockService.getProviderStats();
|
||||
const marketstackStats = stats.get('Marketstack');
|
||||
console.log('Marketstack Stats:', {
|
||||
successCount: marketstackStats.successCount,
|
||||
errorCount: marketstackStats.errorCount,
|
||||
lastError: marketstackStats.lastError
|
||||
});
|
||||
```
|
||||
|
||||
### Handelsregister Integration
|
||||
|
||||
Automate German company data retrieval:
|
||||
|
||||
```typescript
|
||||
// Search for a company
|
||||
const results = await openData.handelsregister.searchCompany("Siemens AG");
|
||||
|
||||
// Get detailed information and documents
|
||||
const details = await openData.handelsregister.getSpecificCompany({
|
||||
court: "Munich",
|
||||
type: "HRB",
|
||||
number: "6684"
|
||||
});
|
||||
|
||||
// Downloaded files include:
|
||||
// - XML data (SI files)
|
||||
// - PDF documents (AD files)
|
||||
for (const file of details.files) {
|
||||
await file.writeToDir('./downloads');
|
||||
}
|
||||
```
|
||||
|
||||
### Combined Data Analysis
|
||||
|
||||
Merge financial and business data:
|
||||
|
||||
```typescript
|
||||
// Find all public German companies (AG)
|
||||
const publicCompanies = await openData.db
|
||||
.collection('businessrecords')
|
||||
.find({ legalForm: 'AG' })
|
||||
.toArray();
|
||||
|
||||
// Enrich with stock data
|
||||
for (const company of publicCompanies) {
|
||||
try {
|
||||
// Map company to ticker (custom logic needed)
|
||||
const ticker = mapCompanyToTicker(company.data.name);
|
||||
|
||||
if (ticker) {
|
||||
const stock = await stockService.getPrice({ ticker });
|
||||
|
||||
// Add financial metrics
|
||||
company.data.stockPrice = stock.price;
|
||||
company.data.marketCap = stock.price * getSharesOutstanding(ticker);
|
||||
company.data.priceChange = stock.changePercent;
|
||||
|
||||
await company.save();
|
||||
}
|
||||
} catch (error) {
|
||||
// Handle missing tickers gracefully
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Stock Service Options
|
||||
|
||||
```typescript
|
||||
const stockService = new StockPriceService({
|
||||
ttl: 60000, // Cache for 1 minute
|
||||
maxEntries: 1000 // Max cached symbols
|
||||
});
|
||||
|
||||
// Marketstack - EOD data, requires API key
|
||||
stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
|
||||
enabled: true,
|
||||
priority: 100,
|
||||
timeout: 10000,
|
||||
retryAttempts: 3,
|
||||
retryDelay: 1000
|
||||
});
|
||||
```
|
||||
|
||||
### MongoDB Setup
|
||||
|
||||
Set environment variables for German business data:
|
||||
|
||||
```env
|
||||
MONGODB_URL=mongodb://localhost:27017
|
||||
MONGODB_NAME=opendata
|
||||
MONGODB_USER=myuser
|
||||
MONGODB_PASS=mypass
|
||||
```
|
||||
|
||||
### Marketstack API Key
|
||||
|
||||
Get your free API key at [marketstack.com](https://marketstack.com) and set it in your environment:
|
||||
|
||||
```env
|
||||
MARKETSTACK_COM_TOKEN=your_api_key_here
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Stock Types
|
||||
|
||||
```typescript
|
||||
interface IStockPrice {
|
||||
ticker: string;
|
||||
price: number;
|
||||
currency: string;
|
||||
change: number;
|
||||
changePercent: number;
|
||||
previousClose: number;
|
||||
timestamp: Date;
|
||||
provider: string;
|
||||
marketState: 'PRE' | 'REGULAR' | 'POST' | 'CLOSED';
|
||||
exchange?: string;
|
||||
exchangeName?: string;
|
||||
}
|
||||
```
|
||||
|
||||
### Key Methods
|
||||
|
||||
**StockPriceService**
|
||||
- `getPrice(request)` - Single stock price with automatic provider selection
|
||||
- `getPrices(request)` - Batch prices (100+ symbols in one request)
|
||||
- `register(provider, config)` - Add data provider with priority and retry config
|
||||
- `checkProvidersHealth()` - Test all providers and return health status
|
||||
- `getProviderStats()` - Get success/error statistics for each provider
|
||||
- `clearCache()` - Clear price cache
|
||||
- `setCacheTTL(ttl)` - Update cache TTL dynamically
|
||||
|
||||
**MarketstackProvider**
|
||||
- ✅ End-of-Day (EOD) data
|
||||
- ✅ 125,000+ tickers across 72+ exchanges worldwide
|
||||
- ✅ Batch fetching support (multiple symbols in one request)
|
||||
- ✅ Comprehensive data: open, high, low, close, volume, splits, dividends
|
||||
- ⚠️ Requires API key (free tier: 100 requests/month)
|
||||
- ⚠️ EOD data only (not real-time)
|
||||
|
||||
**OpenData**
|
||||
- `start()` - Initialize MongoDB connection
|
||||
- `buildInitialDb()` - Import bulk data
|
||||
- `CBusinessRecord` - Business record class
|
||||
- `handelsregister` - Registry automation
|
||||
|
||||
## Provider Architecture
|
||||
|
||||
The library uses a flexible provider system that makes it easy to add new data sources:
|
||||
|
||||
```typescript
|
||||
class MyCustomProvider implements IStockProvider {
|
||||
name = 'My Provider';
|
||||
priority = 50;
|
||||
requiresAuth = true;
|
||||
|
||||
async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
// Your implementation
|
||||
}
|
||||
|
||||
async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
|
||||
// Batch implementation
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
// Health check
|
||||
}
|
||||
|
||||
supportsMarket(market: string): boolean {
|
||||
// Market validation
|
||||
}
|
||||
|
||||
supportsTicker(ticker: string): boolean {
|
||||
// Ticker validation
|
||||
}
|
||||
}
|
||||
|
||||
stockService.register(new MyCustomProvider());
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
- **Batch fetching**: Get 100+ EOD prices in one API request
|
||||
- **Smart caching**: Instant repeated queries with configurable TTL
|
||||
- **Rate limit aware**: Automatic retry logic for API limits
|
||||
- **Concurrent processing**: Handle 1000+ business records/second
|
||||
- **Streaming**: Process GB-sized datasets without memory issues
|
||||
|
||||
## Testing
|
||||
|
||||
Run the comprehensive test suite:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
Test stock provider:
|
||||
|
||||
```bash
|
||||
npx tstest test/test.marketstack.node.ts --verbose
|
||||
```
|
||||
|
||||
Test German business data:
|
||||
|
||||
```bash
|
||||
npx tstest test/test.handelsregister.ts --verbose
|
||||
```
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
|
302
test/test.marketstack.node.ts
Normal file
302
test/test.marketstack.node.ts
Normal file
@@ -0,0 +1,302 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as opendata from '../ts/index.js';
|
||||
import * as paths from '../ts/paths.js';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
|
||||
// Test data
|
||||
const testTickers = ['AAPL', 'MSFT', 'GOOGL'];
|
||||
const invalidTicker = 'INVALID_TICKER_XYZ';
|
||||
|
||||
let stockService: opendata.StockPriceService;
|
||||
let marketstackProvider: opendata.MarketstackProvider;
|
||||
let testQenv: plugins.qenv.Qenv;
|
||||
|
||||
tap.test('should create StockPriceService instance', async () => {
|
||||
stockService = new opendata.StockPriceService({
|
||||
ttl: 30000, // 30 seconds cache
|
||||
maxEntries: 100
|
||||
});
|
||||
expect(stockService).toBeInstanceOf(opendata.StockPriceService);
|
||||
});
|
||||
|
||||
tap.test('should create MarketstackProvider instance', async () => {
|
||||
try {
|
||||
// Create qenv and get API key
|
||||
testQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
|
||||
const apiKey = await testQenv.getEnvVarOnDemand('MARKETSTACK_COM_TOKEN');
|
||||
|
||||
marketstackProvider = new opendata.MarketstackProvider(apiKey, {
|
||||
enabled: true,
|
||||
timeout: 10000,
|
||||
retryAttempts: 2,
|
||||
retryDelay: 500
|
||||
});
|
||||
expect(marketstackProvider).toBeInstanceOf(opendata.MarketstackProvider);
|
||||
expect(marketstackProvider.name).toEqual('Marketstack');
|
||||
expect(marketstackProvider.requiresAuth).toEqual(true);
|
||||
expect(marketstackProvider.priority).toEqual(80);
|
||||
} catch (error) {
|
||||
if (error.message.includes('MARKETSTACK_COM_TOKEN')) {
|
||||
console.log('⚠️ MARKETSTACK_COM_TOKEN not set - skipping Marketstack tests');
|
||||
tap.test('Marketstack token not available', async () => {
|
||||
expect(true).toEqual(true); // Skip gracefully
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should register Marketstack provider with the service', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
stockService.register(marketstackProvider);
|
||||
const providers = stockService.getAllProviders();
|
||||
expect(providers).toContainEqual(marketstackProvider);
|
||||
expect(stockService.getProvider('Marketstack')).toEqual(marketstackProvider);
|
||||
});
|
||||
|
||||
tap.test('should check provider health', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
const health = await stockService.checkProvidersHealth();
|
||||
expect(health.get('Marketstack')).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should fetch single stock price', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
const price = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
expect(price).toHaveProperty('ticker');
|
||||
expect(price).toHaveProperty('price');
|
||||
expect(price).toHaveProperty('currency');
|
||||
expect(price).toHaveProperty('change');
|
||||
expect(price).toHaveProperty('changePercent');
|
||||
expect(price).toHaveProperty('previousClose');
|
||||
expect(price).toHaveProperty('timestamp');
|
||||
expect(price).toHaveProperty('provider');
|
||||
expect(price).toHaveProperty('marketState');
|
||||
|
||||
expect(price.ticker).toEqual('AAPL');
|
||||
expect(price.price).toBeGreaterThan(0);
|
||||
expect(price.provider).toEqual('Marketstack');
|
||||
expect(price.timestamp).toBeInstanceOf(Date);
|
||||
expect(price.marketState).toEqual('CLOSED'); // EOD data
|
||||
|
||||
console.log(`✓ Fetched AAPL: $${price.price} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)`);
|
||||
});
|
||||
|
||||
tap.test('should fetch multiple stock prices', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
const prices = await stockService.getPrices({
|
||||
tickers: testTickers
|
||||
});
|
||||
|
||||
expect(prices).toBeArray();
|
||||
expect(prices.length).toBeGreaterThan(0);
|
||||
expect(prices.length).toBeLessThanOrEqual(testTickers.length);
|
||||
|
||||
for (const price of prices) {
|
||||
expect(testTickers).toContain(price.ticker);
|
||||
expect(price.price).toBeGreaterThan(0);
|
||||
expect(price.provider).toEqual('Marketstack');
|
||||
expect(price.marketState).toEqual('CLOSED');
|
||||
console.log(` ${price.ticker}: $${price.price}`);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should serve cached prices on subsequent requests', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
// First request - should hit the API
|
||||
const firstRequest = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
// Second request - should be served from cache
|
||||
const secondRequest = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
expect(secondRequest.ticker).toEqual(firstRequest.ticker);
|
||||
expect(secondRequest.price).toEqual(firstRequest.price);
|
||||
expect(secondRequest.timestamp).toEqual(firstRequest.timestamp);
|
||||
|
||||
console.log('✓ Cache working correctly');
|
||||
});
|
||||
|
||||
tap.test('should handle invalid ticker gracefully', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await stockService.getPrice({ ticker: invalidTicker });
|
||||
throw new Error('Should have thrown an error for invalid ticker');
|
||||
} catch (error) {
|
||||
expect(error.message).toInclude('Failed to fetch price');
|
||||
console.log('✓ Invalid ticker handled correctly');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should support market checking', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
expect(marketstackProvider.supportsMarket('US')).toEqual(true);
|
||||
expect(marketstackProvider.supportsMarket('UK')).toEqual(true);
|
||||
expect(marketstackProvider.supportsMarket('DE')).toEqual(true);
|
||||
expect(marketstackProvider.supportsMarket('JP')).toEqual(true);
|
||||
expect(marketstackProvider.supportsMarket('INVALID')).toEqual(false);
|
||||
|
||||
console.log('✓ Market support check working');
|
||||
});
|
||||
|
||||
tap.test('should validate ticker format', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
expect(marketstackProvider.supportsTicker('AAPL')).toEqual(true);
|
||||
expect(marketstackProvider.supportsTicker('MSFT')).toEqual(true);
|
||||
expect(marketstackProvider.supportsTicker('BRK.B')).toEqual(true);
|
||||
expect(marketstackProvider.supportsTicker('123456789012')).toEqual(false);
|
||||
expect(marketstackProvider.supportsTicker('invalid@ticker')).toEqual(false);
|
||||
|
||||
console.log('✓ Ticker validation working');
|
||||
});
|
||||
|
||||
tap.test('should get provider statistics', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
const stats = stockService.getProviderStats();
|
||||
const marketstackStats = stats.get('Marketstack');
|
||||
|
||||
expect(marketstackStats).not.toEqual(undefined);
|
||||
expect(marketstackStats.successCount).toBeGreaterThan(0);
|
||||
expect(marketstackStats.errorCount).toBeGreaterThanOrEqual(0);
|
||||
|
||||
console.log(`✓ Provider stats: ${marketstackStats.successCount} successes, ${marketstackStats.errorCount} errors`);
|
||||
});
|
||||
|
||||
tap.test('should test direct provider methods', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('\n🔍 Testing direct provider methods:');
|
||||
|
||||
// Test isAvailable
|
||||
const available = await marketstackProvider.isAvailable();
|
||||
expect(available).toEqual(true);
|
||||
console.log(' ✓ isAvailable() returned true');
|
||||
|
||||
// Test fetchPrice directly
|
||||
const price = await marketstackProvider.fetchPrice({ ticker: 'MSFT' });
|
||||
expect(price.ticker).toEqual('MSFT');
|
||||
expect(price.provider).toEqual('Marketstack');
|
||||
expect(price.price).toBeGreaterThan(0);
|
||||
console.log(` ✓ fetchPrice() for MSFT: $${price.price}`);
|
||||
|
||||
// Test fetchPrices directly
|
||||
const prices = await marketstackProvider.fetchPrices({
|
||||
tickers: ['AAPL', 'GOOGL']
|
||||
});
|
||||
expect(prices.length).toBeGreaterThan(0);
|
||||
console.log(` ✓ fetchPrices() returned ${prices.length} prices`);
|
||||
|
||||
for (const p of prices) {
|
||||
console.log(` ${p.ticker}: $${p.price}`);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should fetch sample EOD data', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('\n📊 Sample EOD Stock Data from Marketstack:');
|
||||
console.log('═'.repeat(65));
|
||||
|
||||
const sampleTickers = [
|
||||
{ ticker: 'AAPL', name: 'Apple Inc.' },
|
||||
{ ticker: 'MSFT', name: 'Microsoft Corp.' },
|
||||
{ ticker: 'GOOGL', name: 'Alphabet Inc.' },
|
||||
{ ticker: 'AMZN', name: 'Amazon.com Inc.' },
|
||||
{ ticker: 'TSLA', name: 'Tesla Inc.' }
|
||||
];
|
||||
|
||||
try {
|
||||
const prices = await marketstackProvider.fetchPrices({
|
||||
tickers: sampleTickers.map(t => t.ticker)
|
||||
});
|
||||
|
||||
const priceMap = new Map(prices.map(p => [p.ticker, p]));
|
||||
|
||||
for (const stock of sampleTickers) {
|
||||
const price = priceMap.get(stock.ticker);
|
||||
if (price) {
|
||||
const changeSymbol = price.change >= 0 ? '↑' : '↓';
|
||||
const changeColor = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
|
||||
const resetColor = '\x1b[0m';
|
||||
|
||||
console.log(
|
||||
`${stock.name.padEnd(20)} ${price.price.toLocaleString('en-US', {
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: 2
|
||||
}).padStart(10)} ${changeColor}${changeSymbol} ${price.change >= 0 ? '+' : ''}${price.change.toFixed(2)} (${price.changePercent >= 0 ? '+' : ''}${price.changePercent.toFixed(2)}%)${resetColor}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('═'.repeat(65));
|
||||
console.log(`Provider: Marketstack (EOD Data)`);
|
||||
console.log(`Last updated: ${new Date().toLocaleString()}\n`);
|
||||
} catch (error) {
|
||||
console.log('Error fetching sample data:', error.message);
|
||||
}
|
||||
|
||||
expect(true).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should clear cache', async () => {
|
||||
if (!marketstackProvider) {
|
||||
console.log('⚠️ Skipping - Marketstack provider not initialized');
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure we have something in cache
|
||||
await stockService.getPrice({ ticker: 'AAPL' });
|
||||
|
||||
// Clear cache
|
||||
stockService.clearCache();
|
||||
console.log('✓ Cache cleared');
|
||||
|
||||
// Next request should hit the API again
|
||||
const price = await stockService.getPrice({ ticker: 'AAPL' });
|
||||
expect(price).not.toEqual(undefined);
|
||||
});
|
||||
|
||||
export default tap.start();
|
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@fin.cx/opendata',
|
||||
version: '1.5.3',
|
||||
description: 'A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.'
|
||||
version: '1.7.0',
|
||||
description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
|
||||
}
|
||||
|
@@ -7,3 +7,4 @@ export * from './classes.stockservice.js';
|
||||
|
||||
// Export providers
|
||||
export * from './providers/provider.yahoo.js';
|
||||
export * from './providers/provider.marketstack.js';
|
@@ -1,3 +1,5 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
|
||||
export interface IStockPrice {
|
||||
ticker: string;
|
||||
price: number;
|
||||
@@ -11,6 +13,10 @@ export interface IStockPrice {
|
||||
exchange?: string;
|
||||
exchangeName?: string;
|
||||
}
|
||||
type CheckStockPrice = plugins.tsclass.typeFest.IsEqual<
|
||||
IStockPrice,
|
||||
plugins.tsclass.finance.IStockPrice
|
||||
>;
|
||||
|
||||
export interface IStockPriceError {
|
||||
ticker: string;
|
||||
|
200
ts/stocks/providers/provider.marketstack.ts
Normal file
200
ts/stocks/providers/provider.marketstack.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import * as plugins from '../../plugins.js';
|
||||
import type { IStockProvider, IProviderConfig } from '../interfaces/provider.js';
|
||||
import type { IStockPrice, IStockQuoteRequest, IStockBatchQuoteRequest } from '../interfaces/stockprice.js';
|
||||
|
||||
/**
|
||||
* Marketstack API v2 Provider
|
||||
* Documentation: https://marketstack.com/documentation_v2
|
||||
*
|
||||
* Features:
|
||||
* - End-of-Day (EOD) stock prices
|
||||
* - Supports 125,000+ tickers across 72+ exchanges worldwide
|
||||
* - Requires API key authentication
|
||||
*
|
||||
* Rate Limits:
|
||||
* - Free Plan: 100 requests/month (EOD only)
|
||||
* - Basic Plan: 10,000 requests/month
|
||||
* - Professional Plan: 100,000 requests/month
|
||||
*
|
||||
* Note: This provider returns EOD data, not real-time prices
|
||||
*/
|
||||
export class MarketstackProvider implements IStockProvider {
|
||||
public name = 'Marketstack';
|
||||
public priority = 80; // Lower than Yahoo (100) due to rate limits and EOD-only data
|
||||
public readonly requiresAuth = true;
|
||||
public readonly rateLimit = {
|
||||
requestsPerMinute: undefined, // No per-minute limit specified
|
||||
requestsPerDay: undefined // Varies by plan
|
||||
};
|
||||
|
||||
private logger = console;
|
||||
private baseUrl = 'https://api.marketstack.com/v2';
|
||||
private apiKey: string;
|
||||
|
||||
constructor(apiKey: string, private config?: IProviderConfig) {
|
||||
if (!apiKey) {
|
||||
throw new Error('API key is required for Marketstack provider');
|
||||
}
|
||||
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch latest EOD price for a single ticker
|
||||
*/
|
||||
public async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
try {
|
||||
const url = `${this.baseUrl}/tickers/${request.ticker}/eod/latest?access_key=${this.apiKey}`;
|
||||
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.timeout(this.config?.timeout || 10000)
|
||||
.get();
|
||||
|
||||
const responseData = await response.json() as any;
|
||||
|
||||
// Check for API errors
|
||||
if (responseData.error) {
|
||||
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
|
||||
}
|
||||
|
||||
// For single ticker endpoint, response is direct object (not wrapped in data field)
|
||||
if (!responseData || !responseData.close) {
|
||||
throw new Error(`No data found for ticker ${request.ticker}`);
|
||||
}
|
||||
|
||||
return this.mapToStockPrice(responseData);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to fetch price for ${request.ticker}:`, error);
|
||||
throw new Error(`Marketstack: Failed to fetch price for ${request.ticker}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch latest EOD prices for multiple tickers
|
||||
*/
|
||||
public async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
|
||||
try {
|
||||
const symbols = request.tickers.join(',');
|
||||
const url = `${this.baseUrl}/eod/latest?access_key=${this.apiKey}&symbols=${symbols}`;
|
||||
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.timeout(this.config?.timeout || 15000)
|
||||
.get();
|
||||
|
||||
const responseData = await response.json() as any;
|
||||
|
||||
// Check for API errors
|
||||
if (responseData.error) {
|
||||
throw new Error(`Marketstack API error: ${responseData.error.message || JSON.stringify(responseData.error)}`);
|
||||
}
|
||||
|
||||
if (!responseData?.data || !Array.isArray(responseData.data)) {
|
||||
throw new Error('Invalid response format from Marketstack API');
|
||||
}
|
||||
|
||||
const prices: IStockPrice[] = [];
|
||||
|
||||
for (const data of responseData.data) {
|
||||
try {
|
||||
prices.push(this.mapToStockPrice(data));
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to parse data for ${data.symbol}:`, error);
|
||||
// Continue processing other tickers
|
||||
}
|
||||
}
|
||||
|
||||
if (prices.length === 0) {
|
||||
throw new Error('No valid price data received from batch request');
|
||||
}
|
||||
|
||||
return prices;
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to fetch batch prices:`, error);
|
||||
throw new Error(`Marketstack: Failed to fetch batch prices: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the Marketstack API is available and accessible
|
||||
*/
|
||||
public async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
// Test with a well-known ticker
|
||||
const url = `${this.baseUrl}/tickers/AAPL/eod/latest?access_key=${this.apiKey}`;
|
||||
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.timeout(5000)
|
||||
.get();
|
||||
|
||||
const responseData = await response.json() as any;
|
||||
|
||||
// Check if we got valid data (not an error)
|
||||
// Single ticker endpoint returns direct object, not wrapped in data field
|
||||
return !responseData.error && responseData.close !== undefined;
|
||||
} catch (error) {
|
||||
this.logger.warn('Marketstack provider is not available:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a market is supported
|
||||
* Marketstack supports 72+ exchanges worldwide
|
||||
*/
|
||||
public supportsMarket(market: string): boolean {
|
||||
// Marketstack has broad international coverage including:
|
||||
// US, UK, DE, FR, JP, CN, HK, AU, CA, IN, etc.
|
||||
const supportedMarkets = [
|
||||
'US', 'UK', 'GB', 'DE', 'FR', 'JP', 'CN', 'HK', 'AU', 'CA',
|
||||
'IN', 'BR', 'MX', 'IT', 'ES', 'NL', 'SE', 'CH', 'NO', 'DK'
|
||||
];
|
||||
return supportedMarkets.includes(market.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a ticker format is supported
|
||||
*/
|
||||
public supportsTicker(ticker: string): boolean {
|
||||
// Basic validation - Marketstack supports most standard ticker formats
|
||||
return /^[A-Z0-9\.\-]{1,10}$/.test(ticker.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Marketstack API response to IStockPrice interface
|
||||
*/
|
||||
private mapToStockPrice(data: any): IStockPrice {
|
||||
if (!data.close) {
|
||||
throw new Error('Missing required price data');
|
||||
}
|
||||
|
||||
// Calculate change and change percent
|
||||
// EOD data: previous close is typically open price of the same day
|
||||
// For better accuracy, we'd need previous day's close, but that requires another API call
|
||||
const currentPrice = data.close;
|
||||
const previousClose = data.open;
|
||||
const change = currentPrice - previousClose;
|
||||
const changePercent = previousClose !== 0 ? (change / previousClose) * 100 : 0;
|
||||
|
||||
// Parse timestamp
|
||||
const timestamp = data.date ? new Date(data.date) : new Date();
|
||||
|
||||
const stockPrice: IStockPrice = {
|
||||
ticker: data.symbol.toUpperCase(),
|
||||
price: currentPrice,
|
||||
currency: data.price_currency || 'USD',
|
||||
change: change,
|
||||
changePercent: changePercent,
|
||||
previousClose: previousClose,
|
||||
timestamp: timestamp,
|
||||
provider: this.name,
|
||||
marketState: 'CLOSED', // EOD data is always for closed markets
|
||||
exchange: data.exchange,
|
||||
exchangeName: data.exchange_code || data.name
|
||||
};
|
||||
|
||||
return stockPrice;
|
||||
}
|
||||
}
|
@@ -20,14 +20,15 @@ export class YahooFinanceProvider implements IStockProvider {
|
||||
public async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
|
||||
try {
|
||||
const url = `${this.baseUrl}/v8/finance/chart/${request.ticker}`;
|
||||
const response = await plugins.smartrequest.getJson(url, {
|
||||
headers: {
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.headers({
|
||||
'User-Agent': this.userAgent
|
||||
},
|
||||
timeout: this.config?.timeout || 10000
|
||||
});
|
||||
})
|
||||
.timeout(this.config?.timeout || 10000)
|
||||
.get();
|
||||
|
||||
const responseData = response.body as any;
|
||||
const responseData = await response.json() as any;
|
||||
|
||||
if (!responseData?.chart?.result?.[0]) {
|
||||
throw new Error(`No data found for ticker ${request.ticker}`);
|
||||
@@ -66,14 +67,15 @@ export class YahooFinanceProvider implements IStockProvider {
|
||||
const symbols = request.tickers.join(',');
|
||||
const url = `${this.baseUrl}/v8/finance/spark?symbols=${symbols}&range=1d&interval=5m`;
|
||||
|
||||
const response = await plugins.smartrequest.getJson(url, {
|
||||
headers: {
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(url)
|
||||
.headers({
|
||||
'User-Agent': this.userAgent
|
||||
},
|
||||
timeout: this.config?.timeout || 15000
|
||||
});
|
||||
})
|
||||
.timeout(this.config?.timeout || 15000)
|
||||
.get();
|
||||
|
||||
const responseData = response.body as any;
|
||||
const responseData = await response.json() as any;
|
||||
const prices: IStockPrice[] = [];
|
||||
|
||||
for (const [ticker, data] of Object.entries(responseData)) {
|
||||
|
Reference in New Issue
Block a user