diff --git a/package.json b/package.json index 30736a1..5bdc06f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartrequest": "^2.0.21", "@push.rocks/smartrx": "^3.0.7", - "@push.rocks/smartstream": "^3.0.27", + "@push.rocks/smartstream": "^3.0.28", "@push.rocks/smartunique": "^3.0.6", "@push.rocks/smarturl": "^3.0.7", "@types/tar-stream": "^3.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a83a2bb..a96bf72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ dependencies: specifier: ^3.0.7 version: 3.0.7 '@push.rocks/smartstream': - specifier: ^3.0.27 - version: 3.0.27 + specifier: ^3.0.28 + version: 3.0.28 '@push.rocks/smartunique': specifier: ^3.0.6 version: 3.0.6 @@ -747,7 +747,7 @@ packages: '@push.rocks/smartpath': 5.0.11 '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartrequest': 2.0.21 - '@push.rocks/smartstream': 3.0.27 + '@push.rocks/smartstream': 3.0.28 '@types/fs-extra': 11.0.3 '@types/glob': 8.1.0 '@types/js-yaml': 4.0.8 @@ -964,8 +964,8 @@ packages: through2: 4.0.2 dev: true - /@push.rocks/smartstream@3.0.27: - resolution: {integrity: sha512-ZIYYS/dVQab+BP2LIP4u4uUn1oqvRvb/vPUabirhyBJwF8VBYC3ssDp9P+oYcuZ3i+Od0G4g1GQuhtFWNv6hgw==} + /@push.rocks/smartstream@3.0.28: + resolution: {integrity: sha512-OYKSjjyQj5h9+bxz4cHfUCpbYGHjB3TvseAY/3gWOrli7neGIPg7ycCDiiUYXfYQuPjof9GMzuBsAy4cpiyFNA==} dependencies: '@push.rocks/lik': 6.0.12 '@push.rocks/smartpromise': 4.0.3 diff --git a/test/test.ts b/test/test.ts index 5722b37..26223f1 100644 --- a/test/test.ts +++ b/test/test.ts @@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => { await testSmartarchive.exportToFs(testPaths.nogitDir); }); -tap.skip.test('should extract a b2zip', async () => { +tap.test('should extract a b2zip', async () => { const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl); await testArchive.exportToFs( @@ -47,4 +47,4 @@ tap.skip.test('should extract a b2zip', async () => { ); }) -tap.start(); +await tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 0afc967..e8acf91 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartarchive', - version: '4.0.16', + version: '4.0.17', description: 'work with archives' } diff --git a/ts/bzip2/index.ts b/ts/bzip2/index.ts index 76d57ac..e42deee 100644 --- a/ts/bzip2/index.ts +++ b/ts/bzip2/index.ts @@ -55,7 +55,7 @@ export function unbzip2Stream() { return new plugins.smartstream.SmartDuplex({ objectMode: true, name: 'bzip2', - // debug: true, + debug: true, highWaterMark: 1, writeFunction: async function (data, streamTools) { // console.log(`got chunk ${counter++}`) diff --git a/ts/classes.archiveanalyzer.ts b/ts/classes.archiveanalyzer.ts index eec05e1..93839ae 100644 --- a/ts/classes.archiveanalyzer.ts +++ b/ts/classes.archiveanalyzer.ts @@ -52,11 +52,10 @@ export class ArchiveAnalyzer { const analyzerstream = new plugins.smartstream.SmartDuplex({ readableObjectMode: true, writeFunction: async (chunkArg: Buffer, streamtools) => { - const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg); - const decompressionStream = await this.getDecompressionStream(fileType?.mime as any); - resultStream.push(chunkArg); if (firstRun) { firstRun = false; + const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg); + const decompressionStream = await this.getDecompressionStream(fileType?.mime as any); const result: IAnalyzedResult = { fileType, isArchive: await this.mimeTypeIsArchive(fileType?.mime), @@ -64,8 +63,9 @@ export class ArchiveAnalyzer { decompressionStream, }; await streamtools.push(result); - return null; } + await resultStream.backpressuredPush(chunkArg); + return null; }, finalFunction: async (tools) => { resultStream.push(null);