From c30fc40e0680e44dfd35fbaccf100e6ebad55a00 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Tue, 31 Dec 2024 14:43:08 +0100 Subject: [PATCH] feat(core): Enhanced data handling and retrieval features, improved usage documentation --- changelog.md | 7 + npmextra.json | 16 +- package.json | 16 +- readme.md | 268 ++++++++++++++++++++++++++++--- ts/00_commitinfo_data.ts | 4 +- ts/classes.businessrecord.ts | 4 + ts/classes.germanbusinessdata.ts | 9 ++ 7 files changed, 295 insertions(+), 29 deletions(-) diff --git a/changelog.md b/changelog.md index 3c450d2..6cfcb06 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,12 @@ # Changelog +## 2024-12-31 - 1.1.0 - feat(core) +Enhanced data handling and retrieval features, improved usage documentation + +- Updated package description and added project keywords in package.json. +- Extended readme with detailed usage examples and class structures. +- Added getBusinessRecordByName function for dynamic business record retrieval. + ## 2024-07-02 - 1.0.3 - fix(core) No new changes detected. Preparing for patch release. diff --git a/npmextra.json b/npmextra.json index a5adcd2..1cb11c9 100644 --- a/npmextra.json +++ b/npmextra.json @@ -5,10 +5,22 @@ "githost": "gitlab.com", "gitscope": "fin.cx", "gitrepo": "opendata", - "description": "open business data", + "description": "A TypeScript-based library for accessing and managing open business data, specifically for German companies.", "npmPackagename": "@fin.cx/opendata", "license": "MIT", - "projectDomain": "fin.cx" + "projectDomain": "fin.cx", + "keywords": [ + "TypeScript", + "open data", + "business data", + "German companies", + "data management", + "business registry", + "npm package", + "database", + "MongoDB", + "automation" + ] } }, "npmci": { diff --git a/package.json b/package.json index 3bc1bd1..8e3caaf 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@fin.cx/opendata", "version": "1.0.3", "private": false, - "description": "open business data", + "description": "A TypeScript-based library for accessing and managing open business data, specifically for German companies.", "main": "dist_ts/index.js", "typings": "dist_ts/index.d.ts", "type": "module", @@ -54,5 +54,17 @@ "cli.js", "npmextra.json", "readme.md" + ], + "keywords": [ + "TypeScript", + "open data", + "business data", + "German companies", + "data management", + "business registry", + "npm package", + "database", + "MongoDB", + "automation" ] -} +} \ No newline at end of file diff --git a/readme.md b/readme.md index 152de30..4bf41ac 100644 --- a/readme.md +++ b/readme.md @@ -1,31 +1,253 @@ +```markdown # @fin.cx/opendata open business data -## Availabililty and Links -* [npmjs.org (npm package)](https://www.npmjs.com/package/@fin.cx/opendata) -* [gitlab.com (source)](https://gitlab.com/fin.cx/opendata) -* [github.com (source mirror)](https://github.com/fin.cx/opendata) -* [docs (typedoc)](https://fin.cx.gitlab.io/opendata/) +## Install -## Status for master +To install the `@fin.cx/opendata` package, you can use npm or yarn as your package manager. Here's how you can do it: -Status Category | Status Badge --- | -- -GitLab Pipelines | [![pipeline status](https://gitlab.com/fin.cx/opendata/badges/master/pipeline.svg)](https://lossless.cloud) -GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/fin.cx/opendata/badges/master/coverage.svg)](https://lossless.cloud) -npm | [![npm downloads per month](https://badgen.net/npm/dy/@fin.cx/opendata)](https://lossless.cloud) -Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/fin.cx/opendata)](https://lossless.cloud) -TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud) -node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/) -Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud) -PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@fin.cx/opendata)](https://lossless.cloud) -PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@fin.cx/opendata)](https://lossless.cloud) -BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@fin.cx/opendata)](https://lossless.cloud) +Using npm: + +```bash +npm install @fin.cx/opendata +``` + +Using yarn: + +```bash +yarn add @fin.cx/opendata +``` ## Usage -Use TypeScript for best in class intellisense -For further information read the linked docs at the top of this readme. -## Legal -> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc) -| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy) +The `@fin.cx/opendata` package offers functionalities for handling open business data, with a primary focus on German business data. Let's explore its capabilities through detailed examples. + +### Setting Up + +#### Importing the Module + +Begin by importing necessary components from the `@fin.cx/opendata` package. You'll also need to set up some environment variables for the MongoDB instance. + +```typescript +import { OpenData } from '@fin.cx/opendata'; + +const startOpenDataInstance = async () => { + const openData = new OpenData(); + + await openData.start(); // Start the open data instance + console.log('OpenData instance started.'); + + // your code here + + await openData.stop(); + console.log('OpenData instance stopped.'); +}; + +startOpenDataInstance().catch(console.error); +``` + +### BusinessRecord Usage + +A `BusinessRecord` is the main entity you'll be working with. Here's how you manage business records. + +#### Creating a New BusinessRecord + +```typescript +import { BusinessRecord } from '@fin.cx/opendata'; + +const createBusinessRecord = async (openData: OpenData) => { + const businessRecord = new openData.CBusinessRecord(); + businessRecord.data.name = "Example Company"; + businessRecord.data.address = "Example Street 1"; + businessRecord.data.postalCode = "12345"; + businessRecord.data.city = "Example City"; + businessRecord.data.country = "Germany"; + businessRecord.data.phone = "+49 123 456789"; + businessRecord.data.email = "contact@example.com"; + businessRecord.data.website = "https://example.com"; + businessRecord.data.businessType = "GmbH"; + businessRecord.data.registrationNumber = "HRB 123456"; + businessRecord.data.registrationCourt = "Munich"; + businessRecord.data.legalForm = "GmbH"; + businessRecord.data.managingDirectors = ["John Doe", "Jane Smith"]; + businessRecord.data.foundingDate = new Date().toISOString(); + businessRecord.data.capital = "50,000 EUR"; + businessRecord.data.purpose = "Tech Solutions"; + businessRecord.data.lastUpdate = new Date().toISOString(); + + await businessRecord.save(); + console.log('BusinessRecord saved:', businessRecord); +}; +``` + +#### Retrieving BusinessRecord + +Retrieve a business record by querying the database. + +```typescript +import { BusinessRecord } from '@fin.cx/opendata'; + +const findBusinessRecord = async (openData: OpenData) => { + const businessRecords = await openData.db.collection('businessrecords').find().toArray(); + console.log('Retrieved Business Records:', businessRecords); +}; +``` + +### Updating Business Data + +The `GermanBusinessData` class handles the specifics of updating and maintaining the data. + +#### Updating German Business Data + +```typescript +import { OpenData } from '@fin.cx/opendata'; + +const updateGermanBusinessData = async (openData: OpenData) => { + await openData.germanBusinesses.update(); + console.log('German business data updated.'); +}; + +startOpenDataInstance() + .then((openData) => { + // Use the instance + return updateGermanBusinessData(openData); + }) + .catch(console.error); +``` + +This function downloads the latest data from the German business data source, processes it, and updates the local database. + +### Detailed Class Structures and Methods + +#### OpenData Class + +The `OpenData` class is the main entry point. + +```typescript +import { OpenData } from '@fin.cx/opendata'; + +class OpenData { + db: plugins.smartdata.SmartdataDb; + germanBusinesses: GermanBusinessData; + + private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir); + + public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord); + + public async start() { + // Initialize smart data DB + this.db = new plugins.smartdata.SmartdataDb({ + mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'), + mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'), + mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'), + mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'), + }); + + await this.db.init(); + this.germanBusinesses = new GermanBusinessData(this); + await this.germanBusinesses.start(); + } + + public async stop() { + // Clean up resources if necessary + } +} +``` + +#### GermanBusinessData Class + +The `GermanBusinessData` class handles the specifics of German business data. + +```typescript +import { OpenData } from '@fin.cx/opendata'; +import * as plugins from './plugins'; +import * as paths from './paths'; + +class GermanBusinessData { + public openDataRef: OpenData; + + constructor(openDataRef: OpenData) { + this.openDataRef = openDataRef; + } + + public async start() { + await this.update(); + } + + public async stop() { + // Stop any ongoing processing + } + + public async update() { + const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; + const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir); + + if (!dataExists) { + await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir); + } + + const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl); + const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles(); + + let totalRecordsCounter = 0; + let nextRest: string = ''; + + jsonlDataStream.pipe( + new plugins.smartstream.SmartDuplex({ + objectMode: true, + writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => { + const readStream = await chunkArg.createReadStream(); + readStream.pipe( + new plugins.smartstream.SmartDuplex({ + objectMode: true, + writeFunction: async (chunkArg: Buffer, streamToolsArg) => { + const currentString = nextRest + chunkArg.toString(); + + const lines = currentString.split('\n'); + nextRest = lines.pop(); + + for (const line of lines) { + let entry: any; + try { + entry = JSON.parse(line); + } catch (err) { + console.error('Error parsing line:', err); + continue; + } + + totalRecordsCounter++; + if (totalRecordsCounter % 10000 === 0) { + console.log(`${totalRecordsCounter} total records.`); + } + + const businessRecord = new this.openDataRef.CBusinessRecord(); + businessRecord.data.name = entry?.name; + + await businessRecord.save(); + } + }, + finalFunction: async (streamToolsArg) => { + if (nextRest) { + try { + JSON.parse(nextRest); + } catch (err) { + console.error('Error parsing final chunk:', err); + } + } + } + }) + ); + }, + }) + ); + } +} +``` + +### Conclusion + +This module is designed to make it easier to manage open business data, especially focusing on German business data. The examples above demonstrate the core functionalities, including starting and stopping the service, managing business records, and updating data. + +As you work with `@fin.cx/opendata`, you’ll discover it offers a robust and flexible approach for working with open business data seamlessly. Happy coding! +``` +undefined \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index b4bccbd..ed4972f 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@fin.cx/opendata', - version: '1.0.3', - description: 'open business data' + version: '1.1.0', + description: 'A TypeScript-based library for accessing and managing open business data, specifically for German companies.' } diff --git a/ts/classes.businessrecord.ts b/ts/classes.businessrecord.ts index 7f11a80..a20111d 100644 --- a/ts/classes.businessrecord.ts +++ b/ts/classes.businessrecord.ts @@ -2,6 +2,10 @@ import * as plugins from './plugins.js'; @plugins.smartdata.Manager() export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc { + + @plugins.smartdata.unI() + id: string; + @plugins.smartdata.svDb() data: { name?: string, diff --git a/ts/classes.germanbusinessdata.ts b/ts/classes.germanbusinessdata.ts index ff44dfc..faab9cf 100644 --- a/ts/classes.germanbusinessdata.ts +++ b/ts/classes.germanbusinessdata.ts @@ -72,4 +72,13 @@ export class GermanBusinessData { }) ); } + + public async getBusinessRecordByName(nameArg: string) { + const businessRecord = await this.openDataRef.CBusinessRecord.getInstance({ + data: { + name: { $regex: `${nameArg}`, $options: "i" } as any, + } + }); + return businessRecord; + } }