Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1c0e04cb0d | |||
| c3f6ef531b | |||
| a67a0993d6 | |||
| bc43e4c44a | |||
| 9b2dcd7377 | |||
| 1eda50ad13 |
20
changelog.md
20
changelog.md
@ -1,5 +1,25 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-01-03 - 1.3.0 - feat(core)
|
||||
Enhanced data handling capabilities and improved company search functionalities.
|
||||
|
||||
- Updated business record handling to support more registration types.
|
||||
- Improved search capabilities for fetching company data with refined registration type matching.
|
||||
- Added robust logging for JSONL data processing with early exit on successful parse.
|
||||
- Reorganized test cases to include specific company data retrieval.
|
||||
|
||||
## 2025-01-02 - 1.2.1 - fix(BusinessRecord)
|
||||
Add missing field registrationType to BusinessRecord data
|
||||
|
||||
- Introduced the 'registrationType' field to the BusinessRecord data schema with possible values 'HRA' or 'HRB'.
|
||||
|
||||
## 2025-01-02 - 1.2.0 - feat(core)
|
||||
Integrate Handelsregister search for company data retrieval
|
||||
|
||||
- Added support for searching company data via Handelsregister.
|
||||
- Replaced GermanBusinessData functionality with JsonlDataProcessor.
|
||||
- Included smartbrowser dependency for handling web requests to Handelsregister.
|
||||
|
||||
## 2025-01-01 - 1.1.5 - fix(GermanBusinessData)
|
||||
Add console log for total records processed at the end of the stream.
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@fin.cx/opendata",
|
||||
"version": "1.1.5",
|
||||
"version": "1.3.0",
|
||||
"private": false,
|
||||
"description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
|
||||
"main": "dist_ts/index.js",
|
||||
@ -19,18 +19,20 @@
|
||||
"@git.zone/tsrun": "^1.3.3",
|
||||
"@git.zone/tstest": "^1.0.90",
|
||||
"@push.rocks/tapbundle": "^5.5.4",
|
||||
"@types/node": "^22.10.2"
|
||||
"@types/node": "^22.10.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"@push.rocks/qenv": "^6.1.0",
|
||||
"@push.rocks/smartarchive": "^4.0.39",
|
||||
"@push.rocks/smartbrowser": "^2.0.8",
|
||||
"@push.rocks/smartdata": "^5.2.10",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.0.23",
|
||||
"@push.rocks/smartpath": "^5.0.18",
|
||||
"@push.rocks/smartpromise": "^4.0.4",
|
||||
"@push.rocks/smartrequest": "^2.0.23",
|
||||
"@push.rocks/smartstream": "^3.2.5"
|
||||
"@push.rocks/smartstream": "^3.2.5",
|
||||
"@tsclass/tsclass": "^4.2.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
||||
102
pnpm-lock.yaml
generated
102
pnpm-lock.yaml
generated
@ -14,6 +14,9 @@ importers:
|
||||
'@push.rocks/smartarchive':
|
||||
specifier: ^4.0.39
|
||||
version: 4.0.39
|
||||
'@push.rocks/smartbrowser':
|
||||
specifier: ^2.0.8
|
||||
version: 2.0.8
|
||||
'@push.rocks/smartdata':
|
||||
specifier: ^5.2.10
|
||||
version: 5.2.10(@aws-sdk/client-sso-oidc@3.716.0(@aws-sdk/client-sts@3.716.0))(@aws-sdk/credential-providers@3.716.0(@aws-sdk/client-sso-oidc@3.716.0(@aws-sdk/client-sts@3.716.0)))(socks@2.8.3)
|
||||
@ -35,6 +38,9 @@ importers:
|
||||
'@push.rocks/smartstream':
|
||||
specifier: ^3.2.5
|
||||
version: 3.2.5
|
||||
'@tsclass/tsclass':
|
||||
specifier: ^4.2.0
|
||||
version: 4.2.0
|
||||
devDependencies:
|
||||
'@git.zone/tsbuild':
|
||||
specifier: ^2.2.0
|
||||
@ -52,8 +58,8 @@ importers:
|
||||
specifier: ^5.5.4
|
||||
version: 5.5.4(@aws-sdk/client-sso-oidc@3.716.0(@aws-sdk/client-sts@3.716.0))(@aws-sdk/credential-providers@3.716.0(@aws-sdk/client-sso-oidc@3.716.0(@aws-sdk/client-sts@3.716.0)))(socks@2.8.3)
|
||||
'@types/node':
|
||||
specifier: ^22.10.2
|
||||
version: 22.10.2
|
||||
specifier: ^22.10.4
|
||||
version: 22.10.4
|
||||
|
||||
packages:
|
||||
|
||||
@ -735,8 +741,8 @@ packages:
|
||||
'@push.rocks/smartarchive@4.0.39':
|
||||
resolution: {integrity: sha512-e8xOOa7h4WlZMhjEd7IjAL/wgLBS3yJ6+Q7eZognHg1cNE/TOZ1kYrAN9eo8xmTtd+37hY9NXayk2JwXdXEvyA==}
|
||||
|
||||
'@push.rocks/smartbrowser@2.0.6':
|
||||
resolution: {integrity: sha512-Ne+KCVhV/DROc1rHRRw59K6h0+LpQAK9fdOUtgDZ7laLPmB/tmnbUh3IuRDNcIY1iVA9pydoobwjnTjVgio9eQ==}
|
||||
'@push.rocks/smartbrowser@2.0.8':
|
||||
resolution: {integrity: sha512-0KWRZj3TuKo/sNwgPbiSE6WL+TMeR19t1JmXBZWh9n8iA2mpc4HhMrQAndEUdRCkx5ofSaHWojIRVFzGChj0Dg==}
|
||||
|
||||
'@push.rocks/smartbucket@3.3.7':
|
||||
resolution: {integrity: sha512-RiOuEtwHJ+HFbV1nlZgh5VuMvP6PXElX6rVe7OSQsyNCBybRQa/d1qDic92+2Ejx852DGeHlyREELQCxd/a/7w==}
|
||||
@ -1340,8 +1346,8 @@ packages:
|
||||
'@types/express-serve-static-core@4.19.6':
|
||||
resolution: {integrity: sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==}
|
||||
|
||||
'@types/express-serve-static-core@5.0.2':
|
||||
resolution: {integrity: sha512-vluaspfvWEtE4vcSDlKRNer52DvOGrB2xv6diXy6UKyKW0lqZiWHGNApSyxOv+8DE5Z27IzVvE7hNkxg7EXIcg==}
|
||||
'@types/express-serve-static-core@5.0.3':
|
||||
resolution: {integrity: sha512-JEhMNwUJt7bw728CydvYzntD0XJeTmDnvwLlbfbAhE7Tbslm/ax6bdIiUwTgeVlZTsJQPwZwKpAkyDtIjsvx3g==}
|
||||
|
||||
'@types/express@4.17.21':
|
||||
resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==}
|
||||
@ -1434,8 +1440,8 @@ packages:
|
||||
'@types/node-forge@1.3.11':
|
||||
resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==}
|
||||
|
||||
'@types/node@22.10.2':
|
||||
resolution: {integrity: sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==}
|
||||
'@types/node@22.10.4':
|
||||
resolution: {integrity: sha512-99l6wv4HEzBQhvaU/UGoeBoCK61SCROQaCCGyQSgX2tEQ3rKkNZ2S7CEWnS/4s1LV+8ODdK21UeyR1fHP2mXug==}
|
||||
|
||||
'@types/parse5@6.0.3':
|
||||
resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==}
|
||||
@ -3790,8 +3796,8 @@ packages:
|
||||
symbol-tree@3.2.4:
|
||||
resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==}
|
||||
|
||||
systeminformation@5.24.1:
|
||||
resolution: {integrity: sha512-zQ5BfdVT7qwhj4bobmAv1EhXBVlXr6nOoS0OlcIigw9WkC+PmEqxLarZIyznDe4uIYmUIc87ahXRa7HGR6EGDQ==}
|
||||
systeminformation@5.24.6:
|
||||
resolution: {integrity: sha512-zxmlzFvPVxlUWmDZX1PK8iUf31/BzrDiAqiTcUwhSGw74D8VWm+ikgBTa38eb5We6o5bZHA4RsTPfYzmDbGvWQ==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]
|
||||
hasBin: true
|
||||
@ -5086,7 +5092,7 @@ snapshots:
|
||||
'@git.zone/tsbundle': 2.1.0
|
||||
'@git.zone/tsrun': 1.3.3
|
||||
'@push.rocks/consolecolor': 2.0.2
|
||||
'@push.rocks/smartbrowser': 2.0.6
|
||||
'@push.rocks/smartbrowser': 2.0.8
|
||||
'@push.rocks/smartdelay': 3.0.5
|
||||
'@push.rocks/smartfile': 11.0.23
|
||||
'@push.rocks/smartlog': 3.0.7
|
||||
@ -5139,7 +5145,7 @@ snapshots:
|
||||
'@jest/schemas': 29.6.3
|
||||
'@types/istanbul-lib-coverage': 2.0.6
|
||||
'@types/istanbul-reports': 3.0.4
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/yargs': 17.0.33
|
||||
chalk: 4.1.2
|
||||
|
||||
@ -5352,7 +5358,7 @@ snapshots:
|
||||
tar-stream: 3.1.7
|
||||
through: 2.3.8
|
||||
|
||||
'@push.rocks/smartbrowser@2.0.6':
|
||||
'@push.rocks/smartbrowser@2.0.8':
|
||||
dependencies:
|
||||
'@push.rocks/smartdelay': 3.0.5
|
||||
'@push.rocks/smartpdf': 3.1.8
|
||||
@ -5603,7 +5609,7 @@ snapshots:
|
||||
'@types/default-gateway': 3.0.1
|
||||
isopen: 1.3.0
|
||||
public-ip: 6.0.2
|
||||
systeminformation: 5.24.1
|
||||
systeminformation: 5.24.6
|
||||
|
||||
'@push.rocks/smartnpm@2.0.4':
|
||||
dependencies:
|
||||
@ -6450,14 +6456,14 @@ snapshots:
|
||||
|
||||
'@types/accepts@1.3.7':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/babel__code-frame@7.0.6': {}
|
||||
|
||||
'@types/body-parser@1.19.5':
|
||||
dependencies:
|
||||
'@types/connect': 3.4.38
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/buffer-json@2.0.3': {}
|
||||
|
||||
@ -6473,17 +6479,17 @@ snapshots:
|
||||
|
||||
'@types/clean-css@4.2.11':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
source-map: 0.6.1
|
||||
|
||||
'@types/co-body@6.1.3':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/qs': 6.9.17
|
||||
|
||||
'@types/connect@3.4.38':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/content-disposition@0.5.8': {}
|
||||
|
||||
@ -6496,11 +6502,11 @@ snapshots:
|
||||
'@types/connect': 3.4.38
|
||||
'@types/express': 5.0.0
|
||||
'@types/keygrip': 1.0.6
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/cors@2.8.17':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/debounce@1.2.4': {}
|
||||
|
||||
@ -6514,14 +6520,14 @@ snapshots:
|
||||
|
||||
'@types/express-serve-static-core@4.19.6':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/qs': 6.9.17
|
||||
'@types/range-parser': 1.2.7
|
||||
'@types/send': 0.17.4
|
||||
|
||||
'@types/express-serve-static-core@5.0.2':
|
||||
'@types/express-serve-static-core@5.0.3':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/qs': 6.9.17
|
||||
'@types/range-parser': 1.2.7
|
||||
'@types/send': 0.17.4
|
||||
@ -6536,7 +6542,7 @@ snapshots:
|
||||
'@types/express@5.0.0':
|
||||
dependencies:
|
||||
'@types/body-parser': 1.19.5
|
||||
'@types/express-serve-static-core': 5.0.2
|
||||
'@types/express-serve-static-core': 5.0.3
|
||||
'@types/qs': 6.9.17
|
||||
'@types/serve-static': 1.15.7
|
||||
|
||||
@ -6546,30 +6552,30 @@ snapshots:
|
||||
|
||||
'@types/from2@2.3.5':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/fs-extra@11.0.4':
|
||||
dependencies:
|
||||
'@types/jsonfile': 6.1.4
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/fs-extra@9.0.13':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/glob@7.2.0':
|
||||
dependencies:
|
||||
'@types/minimatch': 5.1.2
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/glob@8.1.0':
|
||||
dependencies:
|
||||
'@types/minimatch': 5.1.2
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/gunzip-maybe@1.4.2':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/hast@3.0.4':
|
||||
dependencies:
|
||||
@ -6603,7 +6609,7 @@ snapshots:
|
||||
|
||||
'@types/jsonfile@6.1.4':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/keygrip@1.0.6': {}
|
||||
|
||||
@ -6620,7 +6626,7 @@ snapshots:
|
||||
'@types/http-errors': 2.0.4
|
||||
'@types/keygrip': 1.0.6
|
||||
'@types/koa-compose': 3.2.8
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/mdast@4.0.4':
|
||||
dependencies:
|
||||
@ -6638,9 +6644,9 @@ snapshots:
|
||||
|
||||
'@types/node-forge@1.3.11':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/node@22.10.2':
|
||||
'@types/node@22.10.4':
|
||||
dependencies:
|
||||
undici-types: 6.20.0
|
||||
|
||||
@ -6658,19 +6664,19 @@ snapshots:
|
||||
|
||||
'@types/s3rver@3.7.4':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/semver@7.5.8': {}
|
||||
|
||||
'@types/send@0.17.4':
|
||||
dependencies:
|
||||
'@types/mime': 1.3.5
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/serve-static@1.15.7':
|
||||
dependencies:
|
||||
'@types/http-errors': 2.0.4
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/send': 0.17.4
|
||||
|
||||
'@types/sinon-chai@3.2.12':
|
||||
@ -6690,15 +6696,15 @@ snapshots:
|
||||
|
||||
'@types/tar-stream@2.2.3':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/tar-stream@3.1.3':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/through2@2.0.41':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/triple-beam@1.3.5': {}
|
||||
|
||||
@ -6722,7 +6728,7 @@ snapshots:
|
||||
|
||||
'@types/whatwg-url@8.2.2':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
'@types/webidl-conversions': 7.0.3
|
||||
|
||||
'@types/which@2.0.2': {}
|
||||
@ -6731,11 +6737,11 @@ snapshots:
|
||||
|
||||
'@types/ws@7.4.7':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/ws@8.5.13':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
|
||||
'@types/yargs-parser@21.0.3': {}
|
||||
|
||||
@ -6745,7 +6751,7 @@ snapshots:
|
||||
|
||||
'@types/yauzl@2.10.3':
|
||||
dependencies:
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
optional: true
|
||||
|
||||
'@ungap/structured-clone@1.2.1': {}
|
||||
@ -7330,7 +7336,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/cookie': 0.4.1
|
||||
'@types/cors': 2.8.17
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
accepts: 1.3.8
|
||||
base64id: 2.0.0
|
||||
cookie: 0.4.2
|
||||
@ -8038,7 +8044,7 @@ snapshots:
|
||||
jest-util@29.7.0:
|
||||
dependencies:
|
||||
'@jest/types': 29.6.3
|
||||
'@types/node': 22.10.2
|
||||
'@types/node': 22.10.4
|
||||
chalk: 4.1.2
|
||||
ci-info: 3.9.0
|
||||
graceful-fs: 4.2.11
|
||||
@ -9415,7 +9421,7 @@ snapshots:
|
||||
|
||||
symbol-tree@3.2.4: {}
|
||||
|
||||
systeminformation@5.24.1: {}
|
||||
systeminformation@5.24.6: {}
|
||||
|
||||
tar-fs@2.1.1:
|
||||
dependencies:
|
||||
|
||||
23
test/test.ts
23
test/test.ts
@ -12,4 +12,27 @@ tap.test('should start the instance', async () => {
|
||||
await testOpenDataInstance.start();
|
||||
})
|
||||
|
||||
tap.skip.test('should build initial data', async () => {
|
||||
await testOpenDataInstance.buildInitialDb();
|
||||
});
|
||||
|
||||
const resultsSearch = tap.test('should get the data for a company', async () => {
|
||||
const result = await testOpenDataInstance.handelsregister.searchCompany('Volkswagen');
|
||||
console.log(result);
|
||||
return result;
|
||||
});
|
||||
|
||||
tap.test('should get the data for a specific company', async () => {
|
||||
const testCompany = (await resultsSearch.testResultPromise)[21]['germanParsedRegistration'];
|
||||
console.log(`trying to find specific company with:`);
|
||||
console.log(testCompany);
|
||||
const result = await testOpenDataInstance.handelsregister.getSpecificCompany(testCompany);
|
||||
console.log(result);
|
||||
});
|
||||
|
||||
tap.test('should stop the instance', async () => {
|
||||
await testOpenDataInstance.stop();
|
||||
});
|
||||
|
||||
|
||||
tap.start()
|
||||
|
||||
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@fin.cx/opendata',
|
||||
version: '1.1.5',
|
||||
version: '1.3.0',
|
||||
description: 'A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.'
|
||||
}
|
||||
|
||||
@ -1,32 +1,45 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
@plugins.smartdata.Manager()
|
||||
export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<BusinessRecord, BusinessRecord> {
|
||||
|
||||
export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<
|
||||
BusinessRecord,
|
||||
BusinessRecord
|
||||
> {
|
||||
@plugins.smartdata.unI()
|
||||
id: string;
|
||||
|
||||
|
||||
@plugins.smartdata.svDb()
|
||||
data: {
|
||||
name?: string,
|
||||
address?: string,
|
||||
postalCode?: string,
|
||||
city?: string,
|
||||
country?: string,
|
||||
phone?: string,
|
||||
fax?: string,
|
||||
email?: string,
|
||||
website?: string,
|
||||
businessType?: string,
|
||||
registrationNumber?: string,
|
||||
registrationCourt?: string,
|
||||
legalForm?: string,
|
||||
managingDirectors?: string[],
|
||||
boardOfDirectors?: string[],
|
||||
supervisoryBoard?: string[],
|
||||
foundingDate?: string,
|
||||
capital?: string,
|
||||
purpose?: string,
|
||||
lastUpdate?: string
|
||||
name?: string;
|
||||
address?: string;
|
||||
postalCode?: string;
|
||||
city?: string;
|
||||
country?: string;
|
||||
phone?: string;
|
||||
fax?: string;
|
||||
email?: string;
|
||||
website?: string;
|
||||
businessType?: string;
|
||||
registrationId?: string;
|
||||
germanParsedRegistration?: {
|
||||
court?: string;
|
||||
type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR';
|
||||
number?: string;
|
||||
};
|
||||
legalForm?:
|
||||
| 'GmbH'
|
||||
| 'GmbH & Co. KG'
|
||||
| 'AG'
|
||||
| 'LLC'
|
||||
| 'LLP'
|
||||
| 'GmbH & Co. KGaA'
|
||||
| 'GmbH & Co. KGaA, LLC';
|
||||
managingDirectors?: string[];
|
||||
boardOfDirectors?: string[];
|
||||
supervisoryBoard?: string[];
|
||||
foundingDate?: string;
|
||||
capital?: string;
|
||||
purpose?: string;
|
||||
lastUpdate?: string;
|
||||
} = {};
|
||||
}
|
||||
}
|
||||
|
||||
231
ts/classes.handelsregister.ts
Normal file
231
ts/classes.handelsregister.ts
Normal file
@ -0,0 +1,231 @@
|
||||
import type { BusinessRecord } from './classes.businessrecord.js';
|
||||
import type { OpenData } from './classes.main.opendata.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
/**
|
||||
* the HandlesRegister exposed as a class
|
||||
*/
|
||||
export class HandelsRegister {
|
||||
private openDataRef: OpenData;
|
||||
|
||||
public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
|
||||
|
||||
constructor(openDataRef: OpenData) {
|
||||
this.openDataRef = openDataRef;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
await this.smartbrowserInstance.start();
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.smartbrowserInstance.stop();
|
||||
}
|
||||
|
||||
// page stuff
|
||||
public getNewPage = async () => {
|
||||
const page = await this.smartbrowserInstance.headlessBrowser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
await page.goto('https://www.handelsregister.de/');
|
||||
return page;
|
||||
};
|
||||
|
||||
private navigateToPage = async (
|
||||
pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page,
|
||||
pageNameArg: string
|
||||
) => {
|
||||
try {
|
||||
await pageArg.evaluate((pageNameArg2) => {
|
||||
const elements = Array.from(document.querySelectorAll('.ui-menuitem-text > span'));
|
||||
const targetElement = elements.find((el) => el.textContent?.trim() === pageNameArg2);
|
||||
if (targetElement) {
|
||||
(targetElement as HTMLElement).click();
|
||||
}
|
||||
}, pageNameArg);
|
||||
console.log(`Navigated to the ${pageNameArg} page successfully.`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to navigate to the ${pageNameArg} page:`, error);
|
||||
}
|
||||
};
|
||||
|
||||
private waitForResults = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page) => {
|
||||
await pageArg.waitForSelector('#ergebnissForm\\:selectedSuchErgebnisFormTable_data', {
|
||||
timeout: 30000,
|
||||
});
|
||||
|
||||
const businessRecords: BusinessRecord['data'][] = await pageArg.evaluate(() => {
|
||||
const rows = document.querySelectorAll(
|
||||
'#ergebnissForm\\:selectedSuchErgebnisFormTable_data > tr'
|
||||
);
|
||||
const records: BusinessRecord['data'][] = [];
|
||||
|
||||
rows.forEach((row) => {
|
||||
const nameElement = row.querySelector('td.ui-panelgrid-cell span.marginLeft20');
|
||||
const cityElement = row.querySelector('td.ui-panelgrid-cell.sitzSuchErgebnisse span');
|
||||
const statusElement = row.querySelector('td.ui-panelgrid-cell span.verticalText');
|
||||
const registrationCourtElement = row.querySelector(
|
||||
'td.ui-panelgrid-cell.fontTableNameSize'
|
||||
);
|
||||
|
||||
const name = nameElement?.textContent?.trim();
|
||||
const city = cityElement?.textContent?.trim();
|
||||
const status = statusElement?.textContent?.trim();
|
||||
const registrationId = registrationCourtElement?.textContent?.trim();
|
||||
|
||||
// Push parsed data into records array
|
||||
records.push({
|
||||
name,
|
||||
city,
|
||||
registrationId,
|
||||
businessType: status,
|
||||
});
|
||||
});
|
||||
|
||||
return records;
|
||||
});
|
||||
return businessRecords;
|
||||
};
|
||||
|
||||
private clickFindButton = async (pageArg: plugins.smartbrowser.smartpuppeteer.puppeteer.Page) => {
|
||||
try {
|
||||
// Wait for the button with the text "Find" to appear
|
||||
await pageArg.waitForSelector('span.ui-button-text.ui-c', { timeout: 5000 });
|
||||
|
||||
// adjust to 100 results per page
|
||||
await pageArg.select('#form\\:ergebnisseProSeite_input', '100');
|
||||
|
||||
// Locate and click the button using its text
|
||||
await pageArg.evaluate(() => {
|
||||
const buttons = Array.from(document.querySelectorAll('span.ui-button-text.ui-c'));
|
||||
const targetButton = buttons.find((button) => button.textContent?.trim() === 'Find');
|
||||
if (targetButton) {
|
||||
const parentButton = targetButton.closest('button') || targetButton;
|
||||
(parentButton as HTMLElement).click();
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Find button clicked successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or click the "Find" button:', error);
|
||||
}
|
||||
};
|
||||
|
||||
// parsing stuff
|
||||
private async parseGermanRegistration(
|
||||
input: string
|
||||
): Promise<BusinessRecord['data']['germanParsedRegistration']> {
|
||||
const regex = /District court (\p{L}[\p{L}\s-]*?(?:\s*\([\p{L}\s-]+\))?)\s+(HRA|HRB|GnR|VR|PR|GsR)\s+(\d+)/u;
|
||||
const match = input.match(regex);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
court: match[1], // Extracts the court name
|
||||
type: match[2] as 'HRA' | 'HRB', // Extracts the type and ensures it matches the specified types
|
||||
number: match[3], // Extracts the number
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for a company by name
|
||||
*/
|
||||
public async searchCompany(companyNameArg: string) {
|
||||
const page = await this.getNewPage();
|
||||
await this.navigateToPage(page, 'Normal search');
|
||||
|
||||
try {
|
||||
// Wait for the textarea to appear
|
||||
await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
|
||||
|
||||
// Enter text into the textarea using page.evaluate
|
||||
const inputText = companyNameArg;
|
||||
await page.evaluate((text) => {
|
||||
const textarea = document.querySelector<HTMLTextAreaElement>('#form\\:schlagwoerter');
|
||||
if (textarea) {
|
||||
textarea.value = text; // Set the value
|
||||
// Trigger the change event manually if required
|
||||
const event = new Event('change', { bubbles: true });
|
||||
textarea.dispatchEvent(event);
|
||||
}
|
||||
}, inputText);
|
||||
|
||||
console.log('Text entered successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or enter text into the textarea:', error);
|
||||
}
|
||||
|
||||
try {
|
||||
// Wait for the radio button's label to appear
|
||||
await page.waitForSelector('label[for="form:schlagwortOptionen:0"]', { timeout: 5000 });
|
||||
|
||||
// Click the label to select the radio button
|
||||
await page.evaluate(() => {
|
||||
const label = document.querySelector<HTMLLabelElement>(
|
||||
'label[for="form:schlagwortOptionen:0"]'
|
||||
);
|
||||
if (label) {
|
||||
label.click();
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Radio button clicked successfully!');
|
||||
} catch (error) {
|
||||
console.error('Failed to find or click the radio button:', error);
|
||||
}
|
||||
|
||||
await this.clickFindButton(page);
|
||||
|
||||
const businessRecords = await this.waitForResults(page);
|
||||
|
||||
for (const record of businessRecords) {
|
||||
record.germanParsedRegistration = await this.parseGermanRegistration(record.registrationId);
|
||||
}
|
||||
|
||||
await page.close();
|
||||
|
||||
// Finally, we return an object, which triggers a JSON file download
|
||||
return businessRecords;
|
||||
}
|
||||
|
||||
public async getSpecificCompany(companyArg: BusinessRecord['data']['germanParsedRegistration']) {
|
||||
const page = await this.getNewPage();
|
||||
await this.navigateToPage(page, 'Normal search');
|
||||
await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });
|
||||
|
||||
// 1) Type of Register:
|
||||
// Open the dropdown to reveal options
|
||||
await page.waitForSelector('#form\\:registerArt_label');
|
||||
await page.click('#form\\:registerArt_label'); // Open the dropdown
|
||||
|
||||
// Wait for the options and select the one matching companyArg.type
|
||||
await page.waitForSelector('#form\\:registerArt_items'); // Ensure dropdown options are loaded
|
||||
await page.evaluate((type) => {
|
||||
const options = Array.from(document.querySelectorAll('#form\\:registerArt_items li'));
|
||||
const targetOption = options.find((option) => option.textContent?.trim() === type); // Match type dynamically
|
||||
(targetOption as any)?.click();
|
||||
}, companyArg.type); // Pass companyArg.type to the browser context
|
||||
|
||||
// 2) Register number:
|
||||
// Fill in the register number
|
||||
await page.waitForSelector('#form\\:registerNummer');
|
||||
await page.type('#form\\:registerNummer', companyArg.number);
|
||||
|
||||
// 3) Register court:
|
||||
// Open the dropdown for the register court
|
||||
await page.waitForSelector('#form\\:registergericht_label');
|
||||
await page.click('#form\\:registergericht_label'); // Open the dropdown
|
||||
|
||||
// Wait for the options and select the one matching companyArg.court
|
||||
await page.waitForSelector('#form\\:registergericht_items'); // Ensure dropdown options are loaded
|
||||
await page.evaluate((court) => {
|
||||
const options = Array.from(document.querySelectorAll('#form\\:registergericht_items li'));
|
||||
const targetOption = options.find((option) => option.textContent?.trim() === court); // Match court dynamically
|
||||
(targetOption as any)?.click();
|
||||
}, companyArg.court); // Pass companyArg.court to the browser context
|
||||
|
||||
await this.clickFindButton(page);
|
||||
|
||||
const businessRecords = await this.waitForResults(page);
|
||||
console.log(businessRecords);
|
||||
}
|
||||
}
|
||||
@ -2,28 +2,23 @@ import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import type { OpenData } from './classes.main.opendata.js';
|
||||
|
||||
export class GermanBusinessData {
|
||||
export class JsonlDataProcessor {
|
||||
public openDataRef: OpenData;
|
||||
constructor(openDataRefArg: OpenData) {
|
||||
this.openDataRef = openDataRefArg;
|
||||
}
|
||||
|
||||
public async start() {
|
||||
await this.update();
|
||||
}
|
||||
public async stop() {}
|
||||
|
||||
public async update() {
|
||||
// TODO: define a mapper as argument instead of hard-coding it
|
||||
public async processDataFromUrl(dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2') {
|
||||
const done = plugins.smartpromise.defer();
|
||||
const promiseArray: Promise<any>[] = [];
|
||||
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||||
const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
|
||||
if (!dataExists) {
|
||||
await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
|
||||
} else {
|
||||
}
|
||||
|
||||
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
||||
const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
|
||||
promiseArray
|
||||
.push
|
||||
// smartarchive.exportToFs(paths.germanBusinessDataDir, 'de_companies_ocdata.jsonl')
|
||||
@ -49,6 +44,8 @@ export class GermanBusinessData {
|
||||
if (!line) continue;
|
||||
try {
|
||||
entry = JSON.parse(line);
|
||||
console.log(JSON.stringify(entry, null, 2));
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
console.log(line);
|
||||
await plugins.smartdelay.delayFor(10000);
|
||||
@ -60,7 +57,6 @@ export class GermanBusinessData {
|
||||
businessRecord.id = await this.openDataRef.CBusinessRecord.getNewId();
|
||||
businessRecord.data.name = entry.name;
|
||||
await businessRecord.save();
|
||||
// console.log(`stored ${businessRecord.data.name}`);
|
||||
}
|
||||
},
|
||||
finalFunction: async (streamToolsArg) => {
|
||||
@ -1,13 +1,16 @@
|
||||
import { BusinessRecord } from './classes.businessrecord.js';
|
||||
import { GermanBusinessData } from './classes.germanbusinessdata.js';
|
||||
import { HandelsRegister } from './classes.handelsregister.js';
|
||||
import { JsonlDataProcessor } from './classes.jsonldata.js';
|
||||
import * as paths from './paths.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export class OpenData {
|
||||
db: plugins.smartdata.SmartdataDb;
|
||||
germanBusinesses: GermanBusinessData;
|
||||
public db: plugins.smartdata.SmartdataDb;
|
||||
private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
|
||||
|
||||
public jsonLDataProcessor: JsonlDataProcessor;
|
||||
public handelsregister: HandelsRegister;
|
||||
|
||||
public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
|
||||
|
||||
public async start() {
|
||||
@ -18,8 +21,17 @@ export class OpenData {
|
||||
mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
|
||||
});
|
||||
await this.db.init();
|
||||
this.germanBusinesses = new GermanBusinessData(this);
|
||||
await this.germanBusinesses.start();
|
||||
this.jsonLDataProcessor = new JsonlDataProcessor(this);
|
||||
this.handelsregister = new HandelsRegister(this);
|
||||
await this.handelsregister.start();
|
||||
}
|
||||
|
||||
public async buildInitialDb() {
|
||||
await this.jsonLDataProcessor.processDataFromUrl();
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.db.close();
|
||||
await this.handelsregister.stop();
|
||||
}
|
||||
public async stop() {}
|
||||
}
|
||||
@ -8,6 +8,7 @@ export {
|
||||
// @push.rocks scope
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartarchive from '@push.rocks/smartarchive';
|
||||
import * as smartbrowser from '@push.rocks/smartbrowser';
|
||||
import * as smartdata from '@push.rocks/smartdata';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
@ -19,6 +20,7 @@ import * as smartstream from '@push.rocks/smartstream';
|
||||
export {
|
||||
qenv,
|
||||
smartarchive,
|
||||
smartbrowser,
|
||||
smartdata,
|
||||
smartdelay,
|
||||
smartfile,
|
||||
@ -26,4 +28,11 @@ export {
|
||||
smartpromise,
|
||||
smartrequest,
|
||||
smartstream,
|
||||
}
|
||||
}
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export {
|
||||
tsclass,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user