fix(core): update

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@fin.cx/opendata',
+  version: '1.0.2',
+  description: 'open business data'
+}

ts/classes.businessrecord.ts

@@ -0,0 +1,28 @@
+import * as plugins from './plugins.js';
+
+@plugins.smartdata.Manager()
+export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<BusinessRecord, BusinessRecord> {
+  @plugins.smartdata.svDb()
+  data: {
+    name?: string,
+    address?: string,
+    postalCode?: string,
+    city?: string,
+    country?: string,
+    phone?: string,
+    fax?: string,
+    email?: string,
+    website?: string,
+    businessType?: string,
+    registrationNumber?: string,
+    registrationCourt?: string,
+    legalForm?: string,
+    managingDirectors?: string[],
+    boardOfDirectors?: string[],
+    supervisoryBoard?: string[],
+    foundingDate?: string,
+    capital?: string,
+    purpose?: string,
+    lastUpdate?: string
+  } = {};
+}

ts/classes.germanbusinessdata.ts

@@ -0,0 +1,75 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import type { OpenData } from './classes.main.opendata.js';
+
+export class GermanBusinessData {
+  public openDataRef: OpenData;
+  constructor(openDataRefArg: OpenData) {
+    this.openDataRef = openDataRefArg;
+  }
+
+  public async start() {
+    await this.update();
+  }
+  public async stop() {}
+
+  public async update() {
+    const done = plugins.smartpromise.defer();
+    const promiseArray: Promise<any>[] = [];
+    const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
+    const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
+    if (!dataExists) {
+      await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
+    } else {
+    }
+
+    const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
+    promiseArray
+      .push
+      // smartarchive.exportToFs(paths.germanBusinessDataDir, 'de_companies_ocdata.jsonl')
+      ();
+    const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
+    let totalRecordsCounter = 0;
+    let nextRest: string = '';
+    jsonlDataStream.pipe(
+      new plugins.smartstream.SmartDuplex({
+        objectMode: true,
+        writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
+          const readStream = await chunkArg.createReadStream();
+          readStream.pipe(
+            new plugins.smartstream.SmartDuplex({
+              objectMode: true,
+              writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
+                const currentString = nextRest + chunkArg.toString();
+                const lines = currentString.split('\n');
+                nextRest = lines.pop();
+                console.log(`Got another ${lines.length} records.`);
+                for (const line of lines) {
+                  let entry: any;
+                  if (!line) continue;
+                  try {
+                    entry = JSON.parse(line);
+                  } catch (err) {
+                    console.log(line);
+                    await plugins.smartdelay.delayFor(10000);
+                  }
+                  if (!entry) continue;
+                  totalRecordsCounter++;
+                  if (totalRecordsCounter % 10000 === 0) console.log(`${totalRecordsCounter} total records.`);
+                  const businessRecord = new this.openDataRef.CBusinessRecord();
+                  businessRecord.data.name = entry.name;
+                  await businessRecord.save();
+                  // console.log(`stored ${businessRecord.data.name}`);
+                }
+              },
+              finalFunction: async (streamToolsArg) => {
+                if (!nextRest) return;
+                JSON.parse(nextRest);
+              }
+            })
+          );
+        },
+      })
+    );
+  }
+}

ts/classes.main.opendata.ts

@@ -0,0 +1,25 @@
+import { BusinessRecord } from './classes.businessrecord.js';
+import { GermanBusinessData } from './classes.germanbusinessdata.js';
+import * as paths from './paths.js';
+import * as plugins from './plugins.js';
+
+export class OpenData {
+  db: plugins.smartdata.SmartdataDb;
+  germanBusinesses: GermanBusinessData;
+  private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
+  
+  public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
+
+  public async start() {
+    this.db = new plugins.smartdata.SmartdataDb({
+        mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
+        mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
+        mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
+        mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
+    });
+    await this.db.init();
+    this.germanBusinesses = new GermanBusinessData(this);
+    await this.germanBusinesses.start();
+  }
+  public async stop() {}
+}

ts/index.ts

@@ -0,0 +1 @@
+export * from './classes.main.opendata.js';

ts/paths.ts

@@ -0,0 +1,11 @@
+import * as plugins from './plugins.js';
+
+export const packageDir = plugins.path.join(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../'
+);
+
+export const nogitDir = plugins.path.join(packageDir, './.nogit/');
+plugins.smartfile.fs.ensureDirSync(nogitDir);
+
+export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');

ts/plugins.ts

@@ -0,0 +1,29 @@
+// node native scope
+import * as path from 'path';
+
+export {
+  path,
+}
+
+// @push.rocks scope
+import * as qenv from '@push.rocks/qenv';
+import * as smartarchive from '@push.rocks/smartarchive';
+import * as smartdata from '@push.rocks/smartdata';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartfile from '@push.rocks/smartfile';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartstream from '@push.rocks/smartstream';
+
+export {
+  qenv,
+  smartarchive,
+  smartdata,
+  smartdelay,
+  smartfile,
+  smartpath,
+  smartpromise,
+  smartrequest,
+  smartstream,
+}