diff --git a/changelog.md b/changelog.md index 1cc44ee..d7c6fa2 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,16 @@ # Changelog +## 2025-10-31 - 2.0.0 - BREAKING CHANGE(OpenData) +Require explicit directory paths for OpenData (nogit/download/germanBusinessData); remove automatic .nogit creation; update HandelsRegister, JsonlDataProcessor, tests and README. + +- Breaking: OpenData constructor now requires a config object with nogitDir, downloadDir and germanBusinessDataDir. The constructor will throw if these paths are not provided. +- Removed automatic creation/export of .nogit/download/germanBusinessData from ts/paths. OpenData.start now ensures the required directories exist. +- HandelsRegister API changed: constructor now accepts downloadDir and manages its own unique download folder; screenshot and download paths now use the configured downloadDir. +- JsonlDataProcessor now accepts a germanBusinessDataDir parameter and uses it when ensuring/storing data instead of relying on global paths. +- Updated tests to provide explicit path configuration (tests now set testNogitDir, testDownloadDir, testGermanBusinessDataDir and write outputs accordingly) and to use updated constructors and qenv usage. +- Documentation updated (README) to document the breaking change and show examples for required directory configuration when instantiating OpenData. +- Added .claude/settings.local.json for local permissions/config used in development/CI environments. + ## 2025-10-11 - 1.7.0 - feat(stocks) Add Marketstack provider (EOD) with tests, exports and documentation updates diff --git a/readme.md b/readme.md index f11309e..b0ad3b2 100644 --- a/readme.md +++ b/readme.md @@ -4,6 +4,20 @@ Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API. +## ⚠️ Breaking Change in v2.0 + +**Directory paths are now MANDATORY when using German business data features.** The package no longer creates `.nogit/` directories automatically. You must explicitly configure all directory paths when instantiating `OpenData`: + +```typescript +const openData = new OpenData({ + nogitDir: '/path/to/your/data', + downloadDir: '/path/to/your/data/downloads', + germanBusinessDataDir: '/path/to/your/data/germanbusinessdata' +}); +``` + +This change enables the package to work in read-only filesystems (like Deno compiled binaries) and gives you full control over where data is stored. + ## Installation ```bash @@ -54,8 +68,14 @@ Access comprehensive data on German companies: ```typescript import { OpenData } from '@fin.cx/opendata'; +import * as path from 'path'; -const openData = new OpenData(); +// REQUIRED: Configure directory paths +const openData = new OpenData({ + nogitDir: path.join(process.cwd(), '.nogit'), + downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), + germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') +}); await openData.start(); // Create a business record @@ -159,6 +179,17 @@ console.log('Marketstack Stats:', { Automate German company data retrieval: ```typescript +import { OpenData } from '@fin.cx/opendata'; +import * as path from 'path'; + +// Configure paths first +const openData = new OpenData({ + nogitDir: path.join(process.cwd(), '.nogit'), + downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), + germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') +}); +await openData.start(); + // Search for a company const results = await openData.handelsregister.searchCompany("Siemens AG"); @@ -182,6 +213,21 @@ for (const file of details.files) { Merge financial and business data: ```typescript +import { OpenData, StockPriceService, MarketstackProvider } from '@fin.cx/opendata'; +import * as path from 'path'; + +// Configure OpenData with paths +const openData = new OpenData({ + nogitDir: path.join(process.cwd(), '.nogit'), + downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), + germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') +}); +await openData.start(); + +// Setup stock service +const stockService = new StockPriceService({ ttl: 60000, maxEntries: 1000 }); +stockService.register(new MarketstackProvider('YOUR_API_KEY')); + // Find all public German companies (AG) const publicCompanies = await openData.db .collection('businessrecords') @@ -212,6 +258,48 @@ for (const company of publicCompanies) { ## Configuration +### Directory Configuration (Required for German Business Data) + +**All directory paths are mandatory when using `OpenData`.** Here are examples for different environments: + +#### Development Environment +```typescript +import { OpenData } from '@fin.cx/opendata'; +import * as path from 'path'; + +const openData = new OpenData({ + nogitDir: path.join(process.cwd(), '.nogit'), + downloadDir: path.join(process.cwd(), '.nogit', 'downloads'), + germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata') +}); +``` + +#### Production Environment +```typescript +import { OpenData } from '@fin.cx/opendata'; +import * as path from 'path'; + +const openData = new OpenData({ + nogitDir: '/var/lib/myapp/data', + downloadDir: '/var/lib/myapp/data/downloads', + germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata' +}); +``` + +#### Deno Compiled Binaries (or other read-only filesystems) +```typescript +import { OpenData } from '@fin.cx/opendata'; + +// Use OS temp directory or user data directory +const dataDir = Deno.env.get('HOME') + '/.myapp/data'; + +const openData = new OpenData({ + nogitDir: dataDir, + downloadDir: dataDir + '/downloads', + germanBusinessDataDir: dataDir + '/germanbusinessdata' +}); +``` + ### Stock Service Options ```typescript diff --git a/test/test.handelsregister.ts b/test/test.handelsregister.ts index 8771729..f1b5bfe 100644 --- a/test/test.handelsregister.ts +++ b/test/test.handelsregister.ts @@ -1,12 +1,24 @@ import { expect, tap } from '@git.zone/tstest/tapbundle'; import * as opendata from '../ts/index.js' +import * as paths from '../ts/paths.js'; +import * as plugins from '../ts/plugins.js'; import { BusinessRecord } from '../ts/classes.businessrecord.js'; +// Test configuration - explicit paths required +const testNogitDir = plugins.path.join(paths.packageDir, '.nogit'); +const testDownloadDir = plugins.path.join(testNogitDir, 'downloads'); +const testGermanBusinessDataDir = plugins.path.join(testNogitDir, 'germanbusinessdata'); +const testOutputDir = plugins.path.join(testNogitDir, 'testoutput'); + let testOpenDataInstance: opendata.OpenData; tap.test('first test', async () => { - testOpenDataInstance = new opendata.OpenData(); + testOpenDataInstance = new opendata.OpenData({ + nogitDir: testNogitDir, + downloadDir: testDownloadDir, + germanBusinessDataDir: testGermanBusinessDataDir + }); expect(testOpenDataInstance).toBeInstanceOf(opendata.OpenData); }); @@ -28,7 +40,7 @@ tap.test('should get the data for a specific company', async () => { console.log(result); await Promise.all(result.files.map(async (file) => { - await file.writeToDir('./.nogit/testoutput'); + await file.writeToDir(testOutputDir); })); diff --git a/test/test.marketstack.node.ts b/test/test.marketstack.node.ts index dc7b6ca..536adea 100644 --- a/test/test.marketstack.node.ts +++ b/test/test.marketstack.node.ts @@ -3,6 +3,9 @@ import * as opendata from '../ts/index.js'; import * as paths from '../ts/paths.js'; import * as plugins from '../ts/plugins.js'; +// Test configuration - explicit paths required +const testNogitDir = plugins.path.join(paths.packageDir, '.nogit'); + // Test data const testTickers = ['AAPL', 'MSFT', 'GOOGL']; const invalidTicker = 'INVALID_TICKER_XYZ'; @@ -22,7 +25,7 @@ tap.test('should create StockPriceService instance', async () => { tap.test('should create MarketstackProvider instance', async () => { try { // Create qenv and get API key - testQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir); + testQenv = new plugins.qenv.Qenv(paths.packageDir, testNogitDir); const apiKey = await testQenv.getEnvVarOnDemand('MARKETSTACK_COM_TOKEN'); marketstackProvider = new opendata.MarketstackProvider(apiKey, { diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 4ef3af8..0dd6453 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@fin.cx/opendata', - version: '1.7.0', + version: '2.0.0', description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.' } diff --git a/ts/classes.handelsregister.ts b/ts/classes.handelsregister.ts index c52b98a..82b9910 100644 --- a/ts/classes.handelsregister.ts +++ b/ts/classes.handelsregister.ts @@ -1,7 +1,6 @@ import type { BusinessRecord } from './classes.businessrecord.js'; import type { OpenData } from './classes.main.opendata.js'; import * as plugins from './plugins.js'; -import * as paths from './paths.js'; /** * the HandlesRegister exposed as a class @@ -9,13 +8,16 @@ import * as paths from './paths.js'; export class HandelsRegister { private openDataRef: OpenData; private asyncExecutionStack = new plugins.lik.AsyncExecutionStack(); - private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple()); + private downloadDir: string; + private uniqueDowloadFolder: string; // Puppeteer wrapper instance public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser(); - constructor(openDataRef: OpenData) { + constructor(openDataRef: OpenData, downloadDirArg: string) { this.openDataRef = openDataRef; + this.downloadDir = downloadDirArg; + this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple()); } public async start() { @@ -76,7 +78,7 @@ export class HandelsRegister { timeout: 30000, }) .catch(async (err) => { - await pageArg.screenshot({ path: paths.downloadDir + '/error.png' }); + await pageArg.screenshot({ path: this.downloadDir + '/error.png' }); throw err; }); diff --git a/ts/classes.jsonldata.ts b/ts/classes.jsonldata.ts index 55b4478..b8805d8 100644 --- a/ts/classes.jsonldata.ts +++ b/ts/classes.jsonldata.ts @@ -1,5 +1,4 @@ import * as plugins from './plugins.js'; -import * as paths from './paths.js'; import type { OpenData } from './classes.main.opendata.js'; export type SeedEntryType = { @@ -41,8 +40,11 @@ export type SeedEntryType = { }; export class JsonlDataProcessor { + private germanBusinessDataDir: string; public forEachFunction: (entryArg: T) => Promise; - constructor(forEachFunctionArg: typeof this.forEachFunction) { + + constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) { + this.germanBusinessDataDir = germanBusinessDataDirArg; this.forEachFunction = forEachFunctionArg; } @@ -51,9 +53,9 @@ export class JsonlDataProcessor { dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2' ) { const done = plugins.smartpromise.defer(); - const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir); + const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir); if (!dataExists) { - await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir); + await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir); } else { } diff --git a/ts/classes.main.opendata.ts b/ts/classes.main.opendata.ts index d4af1b8..f80b32e 100644 --- a/ts/classes.main.opendata.ts +++ b/ts/classes.main.opendata.ts @@ -4,16 +4,39 @@ import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js'; import * as paths from './paths.js'; import * as plugins from './plugins.js'; +export interface IOpenDataConfig { + downloadDir: string; + germanBusinessDataDir: string; + nogitDir: string; +} + export class OpenData { public db: plugins.smartdata.SmartdataDb; - private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir); + private serviceQenv: plugins.qenv.Qenv; + private config: IOpenDataConfig; public jsonLDataProcessor: JsonlDataProcessor; public handelsregister: HandelsRegister; public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord); + constructor(configArg: IOpenDataConfig) { + if (!configArg) { + throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.'); + } + if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) { + throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).'); + } + this.config = configArg; + this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir); + } + public async start() { + // Ensure configured directories exist + await plugins.smartfile.fs.ensureDir(this.config.nogitDir); + await plugins.smartfile.fs.ensureDir(this.config.downloadDir); + await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir); + this.db = new plugins.smartdata.SmartdataDb({ mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'), mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'), @@ -21,18 +44,21 @@ export class OpenData { mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'), }); await this.db.init(); - this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => { - const businessRecord = new this.CBusinessRecord(); - businessRecord.id = await this.CBusinessRecord.getNewId(); - businessRecord.data.name = entryArg.name; - businessRecord.data.germanParsedRegistration = { - court: entryArg.all_attributes.registered_office, - number: entryArg.all_attributes._registerNummer, - type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB', - }; - await businessRecord.save(); - }); - this.handelsregister = new HandelsRegister(this); + this.jsonLDataProcessor = new JsonlDataProcessor( + this.config.germanBusinessDataDir, + async (entryArg) => { + const businessRecord = new this.CBusinessRecord(); + businessRecord.id = await this.CBusinessRecord.getNewId(); + businessRecord.data.name = entryArg.name; + businessRecord.data.germanParsedRegistration = { + court: entryArg.all_attributes.registered_office, + number: entryArg.all_attributes._registerNummer, + type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB', + }; + await businessRecord.save(); + } + ); + this.handelsregister = new HandelsRegister(this, this.config.downloadDir); await this.handelsregister.start(); } diff --git a/ts/paths.ts b/ts/paths.ts index 7ae78a1..a7e05e1 100644 --- a/ts/paths.ts +++ b/ts/paths.ts @@ -3,13 +3,4 @@ import * as plugins from './plugins.js'; export const packageDir = plugins.path.join( plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../' -); - -export const nogitDir = plugins.path.join(packageDir, './.nogit/'); -plugins.smartfile.fs.ensureDirSync(nogitDir); - -export const downloadDir = plugins.path.join(nogitDir, 'downloads'); -plugins.smartfile.fs.ensureDirSync(downloadDir); - - -export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata'); \ No newline at end of file +); \ No newline at end of file