push.rocks/smartarchive
2
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

fix(core): update

Browse Source
This commit is contained in:
Philipp Kunz 2023-11-14 10:55:19 +01:00
parent ad0352a712
commit c578a3fdc1
6 changed files with 14 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -26,7 +26,7 @@
"@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrequest": "^2.0.21", "@push.rocks/smartrequest": "^2.0.21",
"@push.rocks/smartrx": "^3.0.7", "@push.rocks/smartrx": "^3.0.7",
"@push.rocks/smartstream": "^3.0.27", "@push.rocks/smartstream": "^3.0.28",
"@push.rocks/smartunique": "^3.0.6", "@push.rocks/smartunique": "^3.0.6",
"@push.rocks/smarturl": "^3.0.7", "@push.rocks/smarturl": "^3.0.7",
"@types/tar-stream": "^3.1.3", "@types/tar-stream": "^3.1.3",

10
pnpm-lock.yaml
View File

@ -21,8 +21,8 @@ dependencies:
specifier: ^3.0.7 specifier: ^3.0.7
version: 3.0.7 version: 3.0.7
'@push.rocks/smartstream': '@push.rocks/smartstream':
specifier: ^3.0.27 specifier: ^3.0.28
version: 3.0.27 version: 3.0.28
'@push.rocks/smartunique': '@push.rocks/smartunique':
specifier: ^3.0.6 specifier: ^3.0.6
version: 3.0.6 version: 3.0.6
@ -747,7 +747,7 @@ packages:
'@push.rocks/smartpath': 5.0.11 '@push.rocks/smartpath': 5.0.11
'@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartpromise': 4.0.3
'@push.rocks/smartrequest': 2.0.21 '@push.rocks/smartrequest': 2.0.21
'@push.rocks/smartstream': 3.0.27 '@push.rocks/smartstream': 3.0.28
'@types/fs-extra': 11.0.3 '@types/fs-extra': 11.0.3
'@types/glob': 8.1.0 '@types/glob': 8.1.0
'@types/js-yaml': 4.0.8 '@types/js-yaml': 4.0.8
@ -964,8 +964,8 @@ packages:
through2: 4.0.2 through2: 4.0.2
dev: true dev: true
/@push.rocks/smartstream@3.0.27: /@push.rocks/smartstream@3.0.28:
resolution: {integrity: sha512-ZIYYS/dVQab+BP2LIP4u4uUn1oqvRvb/vPUabirhyBJwF8VBYC3ssDp9P+oYcuZ3i+Od0G4g1GQuhtFWNv6hgw==} resolution: {integrity: sha512-OYKSjjyQj5h9+bxz4cHfUCpbYGHjB3TvseAY/3gWOrli7neGIPg7ycCDiiUYXfYQuPjof9GMzuBsAy4cpiyFNA==}
dependencies: dependencies:
'@push.rocks/lik': 6.0.12 '@push.rocks/lik': 6.0.12
'@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartpromise': 4.0.3

4
test/test.ts
View File

@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => {
await testSmartarchive.exportToFs(testPaths.nogitDir); await testSmartarchive.exportToFs(testPaths.nogitDir);
}); });
tap.skip.test('should extract a b2zip', async () => { tap.test('should extract a b2zip', async () => {
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl); const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
await testArchive.exportToFs( await testArchive.exportToFs(
@ -47,4 +47,4 @@ tap.skip.test('should extract a b2zip', async () => {
); );
}) })
tap.start(); await tap.start();

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartarchive', name: '@push.rocks/smartarchive',
version: '4.0.16', version: '4.0.17',
description: 'work with archives' description: 'work with archives'
} }

2
ts/bzip2/index.ts
View File

@ -55,7 +55,7 @@ export function unbzip2Stream() {
return new plugins.smartstream.SmartDuplex({ return new plugins.smartstream.SmartDuplex({
objectMode: true, objectMode: true,
name: 'bzip2', name: 'bzip2',
// debug: true, debug: true,
highWaterMark: 1, highWaterMark: 1,
writeFunction: async function (data, streamTools) { writeFunction: async function (data, streamTools) {
// console.log(`got chunk ${counter++}`) // console.log(`got chunk ${counter++}`)

8
ts/classes.archiveanalyzer.ts
View File

@ -52,11 +52,10 @@ export class ArchiveAnalyzer {
const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({ const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
readableObjectMode: true, readableObjectMode: true,
writeFunction: async (chunkArg: Buffer, streamtools) => { writeFunction: async (chunkArg: Buffer, streamtools) => {
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
resultStream.push(chunkArg);
if (firstRun) { if (firstRun) {
firstRun = false; firstRun = false;
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
const result: IAnalyzedResult = { const result: IAnalyzedResult = {
fileType, fileType,
isArchive: await this.mimeTypeIsArchive(fileType?.mime), isArchive: await this.mimeTypeIsArchive(fileType?.mime),
@ -64,8 +63,9 @@ export class ArchiveAnalyzer {
decompressionStream, decompressionStream,
}; };
await streamtools.push(result); await streamtools.push(result);
return null;
} }
await resultStream.backpressuredPush(chunkArg);
return null;
}, },
finalFunction: async (tools) => { finalFunction: async (tools) => {
resultStream.push(null); resultStream.push(null);