fix(core): update

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

.gitignore

@@ -0,0 +1,20 @@
+.nogit/
+
+# artifacts
+coverage/
+public/
+pages/
+
+# installs
+node_modules/
+
+# caches
+.yarn/
+.cache/
+.rpt2_cache
+
+# builds
+dist/
+dist_*/
+
+# custom

.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "command": "npm test",
+      "name": "Run npm test",
+      "request": "launch",
+      "type": "node-terminal"
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,26 @@
+{
+  "json.schemas": [
+    {
+      "fileMatch": ["/npmextra.json"],
+      "schema": {
+        "type": "object",
+        "properties": {
+          "npmci": {
+            "type": "object",
+            "description": "settings for npmci"
+          },
+          "gitzone": {
+            "type": "object",
+            "description": "settings for gitzone",
+            "properties": {
+              "projectType": {
+                "type": "string",
+                "enum": ["website", "element", "service", "npm", "wcc"]
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

npmextra.json

@@ -0,0 +1,18 @@
+{
+  "gitzone": {
+    "projectType": "npm",
+    "module": {
+      "githost": "gitlab.com",
+      "gitscope": "fin.cx",
+      "gitrepo": "opendata",
+      "description": "open business data",
+      "npmPackagename": "@fin.cx/opendata",
+      "license": "MIT",
+      "projectDomain": "fin.cx"
+    }
+  },
+  "npmci": {
+    "npmGlobalTools": [],
+    "npmAccessLevel": "public"
+  }
+}

package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@fin.cx/opendata",
+  "version": "1.0.1",
+  "private": false,
+  "description": "open business data",
+  "main": "dist_ts/index.js",
+  "typings": "dist_ts/index.d.ts",
+  "type": "module",
+  "author": "Task Venture Capital GmbH",
+  "license": "MIT",
+  "scripts": {
+    "test": "(tstest test/ --web)",
+    "build": "(tsbuild --web --allowimplicitany)",
+    "buildDocs": "(tsdoc)"
+  },
+  "devDependencies": {
+    "@git.zone/tsbuild": "^2.1.25",
+    "@git.zone/tsbundle": "^2.0.5",
+    "@git.zone/tsrun": "^1.2.46",
+    "@git.zone/tstest": "^1.0.84",
+    "@push.rocks/tapbundle": "^5.0.15",
+    "@types/node": "^20.9.0"
+  },
+  "dependencies": {
+    "@push.rocks/qenv": "^6.0.4",
+    "@push.rocks/smartarchive": "^4.0.19",
+    "@push.rocks/smartdata": "^5.0.33",
+    "@push.rocks/smartdelay": "^3.0.5",
+    "@push.rocks/smartfile": "^11.0.0",
+    "@push.rocks/smartpath": "^5.0.11",
+    "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartrequest": "^2.0.21",
+    "@push.rocks/smartstream": "^3.0.30"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/fin.cx/opendata.git"
+  },
+  "bugs": {
+    "url": "https://gitlab.com/fin.cx/opendata/issues"
+  },
+  "homepage": "https://gitlab.com/fin.cx/opendata#readme",
+  "browserslist": [
+    "last 1 chrome versions"
+  ],
+  "files": [
+    "ts/**/*",
+    "ts_web/**/*",
+    "dist/**/*",
+    "dist_*/**/*",
+    "dist_ts/**/*",
+    "dist_ts_web/**/*",
+    "assets/**/*",
+    "cli.js",
+    "npmextra.json",
+    "readme.md"
+  ]
+}

qenv.yml

@@ -0,0 +1,5 @@
+required:
+  - MONGODB_URL
+  - MONGODB_NAME
+  - MONGODB_USER
+  - MONGODB_PASS

readme.md

@@ -0,0 +1,31 @@
+# @fin.cx/opendata
+open business data
+
+## Availabililty and Links
+* [npmjs.org (npm package)](https://www.npmjs.com/package/@fin.cx/opendata)
+* [gitlab.com (source)](https://gitlab.com/fin.cx/opendata)
+* [github.com (source mirror)](https://github.com/fin.cx/opendata)
+* [docs (typedoc)](https://fin.cx.gitlab.io/opendata/)
+
+## Status for master
+
+Status Category | Status Badge
+-- | --
+GitLab Pipelines | [![pipeline status](https://gitlab.com/fin.cx/opendata/badges/master/pipeline.svg)](https://lossless.cloud)
+GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/fin.cx/opendata/badges/master/coverage.svg)](https://lossless.cloud)
+npm | [![npm downloads per month](https://badgen.net/npm/dy/@fin.cx/opendata)](https://lossless.cloud)
+Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/fin.cx/opendata)](https://lossless.cloud)
+TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
+node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
+PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@fin.cx/opendata)](https://lossless.cloud)
+PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@fin.cx/opendata)](https://lossless.cloud)
+BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@fin.cx/opendata)](https://lossless.cloud)
+
+## Usage
+Use TypeScript for best in class intellisense
+For further information read the linked docs at the top of this readme.
+
+## Legal
+> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
+| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)

test/test.ts

@@ -0,0 +1,15 @@
+import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import * as opendata from '../ts/index.js'
+
+let testOpenDataInstance: opendata.OpenData;
+
+tap.test('first test', async () => {
+  testOpenDataInstance = new opendata.OpenData();
+  expect(testOpenDataInstance).toBeInstanceOf(opendata.OpenData);
+});
+
+tap.test('should start the instance', async () => {
+  await testOpenDataInstance.start();
+})
+
+tap.start()

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@fin.cx/opendata',
+  version: '1.0.2',
+  description: 'open business data'
+}

ts/classes.businessrecord.ts

@@ -0,0 +1,28 @@
+import * as plugins from './plugins.js';
+
+@plugins.smartdata.Manager()
+export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<BusinessRecord, BusinessRecord> {
+  @plugins.smartdata.svDb()
+  data: {
+    name?: string,
+    address?: string,
+    postalCode?: string,
+    city?: string,
+    country?: string,
+    phone?: string,
+    fax?: string,
+    email?: string,
+    website?: string,
+    businessType?: string,
+    registrationNumber?: string,
+    registrationCourt?: string,
+    legalForm?: string,
+    managingDirectors?: string[],
+    boardOfDirectors?: string[],
+    supervisoryBoard?: string[],
+    foundingDate?: string,
+    capital?: string,
+    purpose?: string,
+    lastUpdate?: string
+  } = {};
+}

ts/classes.germanbusinessdata.ts

@@ -0,0 +1,75 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import type { OpenData } from './classes.main.opendata.js';
+
+export class GermanBusinessData {
+  public openDataRef: OpenData;
+  constructor(openDataRefArg: OpenData) {
+    this.openDataRef = openDataRefArg;
+  }
+
+  public async start() {
+    await this.update();
+  }
+  public async stop() {}
+
+  public async update() {
+    const done = plugins.smartpromise.defer();
+    const promiseArray: Promise<any>[] = [];
+    const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
+    const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
+    if (!dataExists) {
+      await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
+    } else {
+    }
+
+    const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
+    promiseArray
+      .push
+      // smartarchive.exportToFs(paths.germanBusinessDataDir, 'de_companies_ocdata.jsonl')
+      ();
+    const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
+    let totalRecordsCounter = 0;
+    let nextRest: string = '';
+    jsonlDataStream.pipe(
+      new plugins.smartstream.SmartDuplex({
+        objectMode: true,
+        writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
+          const readStream = await chunkArg.createReadStream();
+          readStream.pipe(
+            new plugins.smartstream.SmartDuplex({
+              objectMode: true,
+              writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
+                const currentString = nextRest + chunkArg.toString();
+                const lines = currentString.split('\n');
+                nextRest = lines.pop();
+                console.log(`Got another ${lines.length} records.`);
+                for (const line of lines) {
+                  let entry: any;
+                  if (!line) continue;
+                  try {
+                    entry = JSON.parse(line);
+                  } catch (err) {
+                    console.log(line);
+                    await plugins.smartdelay.delayFor(10000);
+                  }
+                  if (!entry) continue;
+                  totalRecordsCounter++;
+                  if (totalRecordsCounter % 10000 === 0) console.log(`${totalRecordsCounter} total records.`);
+                  const businessRecord = new this.openDataRef.CBusinessRecord();
+                  businessRecord.data.name = entry.name;
+                  await businessRecord.save();
+                  // console.log(`stored ${businessRecord.data.name}`);
+                }
+              },
+              finalFunction: async (streamToolsArg) => {
+                if (!nextRest) return;
+                JSON.parse(nextRest);
+              }
+            })
+          );
+        },
+      })
+    );
+  }
+}

ts/classes.main.opendata.ts

@@ -0,0 +1,25 @@
+import { BusinessRecord } from './classes.businessrecord.js';
+import { GermanBusinessData } from './classes.germanbusinessdata.js';
+import * as paths from './paths.js';
+import * as plugins from './plugins.js';
+
+export class OpenData {
+  db: plugins.smartdata.SmartdataDb;
+  germanBusinesses: GermanBusinessData;
+  private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
+  
+  public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
+
+  public async start() {
+    this.db = new plugins.smartdata.SmartdataDb({
+        mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
+        mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
+        mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
+        mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
+    });
+    await this.db.init();
+    this.germanBusinesses = new GermanBusinessData(this);
+    await this.germanBusinesses.start();
+  }
+  public async stop() {}
+}

ts/index.ts

@@ -0,0 +1 @@
+export * from './classes.main.opendata.js';

ts/paths.ts

@@ -0,0 +1,11 @@
+import * as plugins from './plugins.js';
+
+export const packageDir = plugins.path.join(
+  plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
+  '../'
+);
+
+export const nogitDir = plugins.path.join(packageDir, './.nogit/');
+plugins.smartfile.fs.ensureDirSync(nogitDir);
+
+export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');

ts/plugins.ts

@@ -0,0 +1,29 @@
+// node native scope
+import * as path from 'path';
+
+export {
+  path,
+}
+
+// @push.rocks scope
+import * as qenv from '@push.rocks/qenv';
+import * as smartarchive from '@push.rocks/smartarchive';
+import * as smartdata from '@push.rocks/smartdata';
+import * as smartdelay from '@push.rocks/smartdelay';
+import * as smartfile from '@push.rocks/smartfile';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartstream from '@push.rocks/smartstream';
+
+export {
+  qenv,
+  smartarchive,
+  smartdata,
+  smartdelay,
+  smartfile,
+  smartpath,
+  smartpromise,
+  smartrequest,
+  smartstream,
+}

tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "useDefineForClassFields": false,
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
+}