253 lines
7.6 KiB
Markdown
253 lines
7.6 KiB
Markdown
```markdown
|
||
# @fin.cx/opendata
|
||
open business data
|
||
|
||
## Install
|
||
|
||
To install the `@fin.cx/opendata` package, you can use npm or yarn as your package manager. Here's how you can do it:
|
||
|
||
Using npm:
|
||
|
||
```bash
|
||
npm install @fin.cx/opendata
|
||
```
|
||
|
||
Using yarn:
|
||
|
||
```bash
|
||
yarn add @fin.cx/opendata
|
||
```
|
||
|
||
## Usage
|
||
|
||
The `@fin.cx/opendata` package offers functionalities for handling open business data, with a primary focus on German business data. Let's explore its capabilities through detailed examples.
|
||
|
||
### Setting Up
|
||
|
||
#### Importing the Module
|
||
|
||
Begin by importing necessary components from the `@fin.cx/opendata` package. You'll also need to set up some environment variables for the MongoDB instance.
|
||
|
||
```typescript
|
||
import { OpenData } from '@fin.cx/opendata';
|
||
|
||
const startOpenDataInstance = async () => {
|
||
const openData = new OpenData();
|
||
|
||
await openData.start(); // Start the open data instance
|
||
console.log('OpenData instance started.');
|
||
|
||
// your code here
|
||
|
||
await openData.stop();
|
||
console.log('OpenData instance stopped.');
|
||
};
|
||
|
||
startOpenDataInstance().catch(console.error);
|
||
```
|
||
|
||
### BusinessRecord Usage
|
||
|
||
A `BusinessRecord` is the main entity you'll be working with. Here's how you manage business records.
|
||
|
||
#### Creating a New BusinessRecord
|
||
|
||
```typescript
|
||
import { BusinessRecord } from '@fin.cx/opendata';
|
||
|
||
const createBusinessRecord = async (openData: OpenData) => {
|
||
const businessRecord = new openData.CBusinessRecord();
|
||
businessRecord.data.name = "Example Company";
|
||
businessRecord.data.address = "Example Street 1";
|
||
businessRecord.data.postalCode = "12345";
|
||
businessRecord.data.city = "Example City";
|
||
businessRecord.data.country = "Germany";
|
||
businessRecord.data.phone = "+49 123 456789";
|
||
businessRecord.data.email = "contact@example.com";
|
||
businessRecord.data.website = "https://example.com";
|
||
businessRecord.data.businessType = "GmbH";
|
||
businessRecord.data.registrationNumber = "HRB 123456";
|
||
businessRecord.data.registrationCourt = "Munich";
|
||
businessRecord.data.legalForm = "GmbH";
|
||
businessRecord.data.managingDirectors = ["John Doe", "Jane Smith"];
|
||
businessRecord.data.foundingDate = new Date().toISOString();
|
||
businessRecord.data.capital = "50,000 EUR";
|
||
businessRecord.data.purpose = "Tech Solutions";
|
||
businessRecord.data.lastUpdate = new Date().toISOString();
|
||
|
||
await businessRecord.save();
|
||
console.log('BusinessRecord saved:', businessRecord);
|
||
};
|
||
```
|
||
|
||
#### Retrieving BusinessRecord
|
||
|
||
Retrieve a business record by querying the database.
|
||
|
||
```typescript
|
||
import { BusinessRecord } from '@fin.cx/opendata';
|
||
|
||
const findBusinessRecord = async (openData: OpenData) => {
|
||
const businessRecords = await openData.db.collection<BusinessRecord>('businessrecords').find().toArray();
|
||
console.log('Retrieved Business Records:', businessRecords);
|
||
};
|
||
```
|
||
|
||
### Updating Business Data
|
||
|
||
The `GermanBusinessData` class handles the specifics of updating and maintaining the data.
|
||
|
||
#### Updating German Business Data
|
||
|
||
```typescript
|
||
import { OpenData } from '@fin.cx/opendata';
|
||
|
||
const updateGermanBusinessData = async (openData: OpenData) => {
|
||
await openData.germanBusinesses.update();
|
||
console.log('German business data updated.');
|
||
};
|
||
|
||
startOpenDataInstance()
|
||
.then((openData) => {
|
||
// Use the instance
|
||
return updateGermanBusinessData(openData);
|
||
})
|
||
.catch(console.error);
|
||
```
|
||
|
||
This function downloads the latest data from the German business data source, processes it, and updates the local database.
|
||
|
||
### Detailed Class Structures and Methods
|
||
|
||
#### OpenData Class
|
||
|
||
The `OpenData` class is the main entry point.
|
||
|
||
```typescript
|
||
import { OpenData } from '@fin.cx/opendata';
|
||
|
||
class OpenData {
|
||
db: plugins.smartdata.SmartdataDb;
|
||
germanBusinesses: GermanBusinessData;
|
||
|
||
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
|
||
|
||
public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
|
||
|
||
public async start() {
|
||
// Initialize smart data DB
|
||
this.db = new plugins.smartdata.SmartdataDb({
|
||
mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
|
||
mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
|
||
mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
|
||
mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
|
||
});
|
||
|
||
await this.db.init();
|
||
this.germanBusinesses = new GermanBusinessData(this);
|
||
await this.germanBusinesses.start();
|
||
}
|
||
|
||
public async stop() {
|
||
// Clean up resources if necessary
|
||
}
|
||
}
|
||
```
|
||
|
||
#### GermanBusinessData Class
|
||
|
||
The `GermanBusinessData` class handles the specifics of German business data.
|
||
|
||
```typescript
|
||
import { OpenData } from '@fin.cx/opendata';
|
||
import * as plugins from './plugins';
|
||
import * as paths from './paths';
|
||
|
||
class GermanBusinessData {
|
||
public openDataRef: OpenData;
|
||
|
||
constructor(openDataRef: OpenData) {
|
||
this.openDataRef = openDataRef;
|
||
}
|
||
|
||
public async start() {
|
||
await this.update();
|
||
}
|
||
|
||
public async stop() {
|
||
// Stop any ongoing processing
|
||
}
|
||
|
||
public async update() {
|
||
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
|
||
|
||
if (!dataExists) {
|
||
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
|
||
}
|
||
|
||
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
||
const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
|
||
|
||
let totalRecordsCounter = 0;
|
||
let nextRest: string = '';
|
||
|
||
jsonlDataStream.pipe(
|
||
new plugins.smartstream.SmartDuplex({
|
||
objectMode: true,
|
||
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
|
||
const readStream = await chunkArg.createReadStream();
|
||
readStream.pipe(
|
||
new plugins.smartstream.SmartDuplex({
|
||
objectMode: true,
|
||
writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
|
||
const currentString = nextRest + chunkArg.toString();
|
||
|
||
const lines = currentString.split('\n');
|
||
nextRest = lines.pop();
|
||
|
||
for (const line of lines) {
|
||
let entry: any;
|
||
try {
|
||
entry = JSON.parse(line);
|
||
} catch (err) {
|
||
console.error('Error parsing line:', err);
|
||
continue;
|
||
}
|
||
|
||
totalRecordsCounter++;
|
||
if (totalRecordsCounter % 10000 === 0) {
|
||
console.log(`${totalRecordsCounter} total records.`);
|
||
}
|
||
|
||
const businessRecord = new this.openDataRef.CBusinessRecord();
|
||
businessRecord.data.name = entry?.name;
|
||
|
||
await businessRecord.save();
|
||
}
|
||
},
|
||
finalFunction: async (streamToolsArg) => {
|
||
if (nextRest) {
|
||
try {
|
||
JSON.parse(nextRest);
|
||
} catch (err) {
|
||
console.error('Error parsing final chunk:', err);
|
||
}
|
||
}
|
||
}
|
||
})
|
||
);
|
||
},
|
||
})
|
||
);
|
||
}
|
||
}
|
||
```
|
||
|
||
### Conclusion
|
||
|
||
This module is designed to make it easier to manage open business data, especially focusing on German business data. The examples above demonstrate the core functionalities, including starting and stopping the service, managing business records, and updating data.
|
||
|
||
As you work with `@fin.cx/opendata`, you’ll discover it offers a robust and flexible approach for working with open business data seamlessly. Happy coding!
|
||
```
|
||
undefined |