4.0.17

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartarchive",
-  "version": "4.0.13",
+  "version": "4.0.17",
   "description": "work with archives",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -26,7 +26,7 @@
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.21",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.0.25",
+    "@push.rocks/smartstream": "^3.0.28",
     "@push.rocks/smartunique": "^3.0.6",
     "@push.rocks/smarturl": "^3.0.7",
     "@types/tar-stream": "^3.1.3",

pnpm-lock.yaml

@@ -21,8 +21,8 @@ dependencies:
     specifier: ^3.0.7
     version: 3.0.7
   '@push.rocks/smartstream':
-    specifier: ^3.0.25
-    version: 3.0.25
+    specifier: ^3.0.28
+    version: 3.0.28
   '@push.rocks/smartunique':
     specifier: ^3.0.6
     version: 3.0.6
@@ -747,7 +747,7 @@ packages:
       '@push.rocks/smartpath': 5.0.11
       '@push.rocks/smartpromise': 4.0.3
       '@push.rocks/smartrequest': 2.0.21
-      '@push.rocks/smartstream': 3.0.25
+      '@push.rocks/smartstream': 3.0.28
       '@types/fs-extra': 11.0.3
       '@types/glob': 8.1.0
       '@types/js-yaml': 4.0.8
@@ -964,8 +964,8 @@ packages:
       through2: 4.0.2
     dev: true
 
-  /@push.rocks/smartstream@3.0.25:
-    resolution: {integrity: sha512-rnZ4Xbuhf6CbgRBjV3o7ddSIR2baWsbeEpV39meia2CfsyY9W/WKxBQrWJbrrv8w165ksSWm9Y9Vy0PmNKr5rQ==}
+  /@push.rocks/smartstream@3.0.28:
+    resolution: {integrity: sha512-OYKSjjyQj5h9+bxz4cHfUCpbYGHjB3TvseAY/3gWOrli7neGIPg7ycCDiiUYXfYQuPjof9GMzuBsAy4cpiyFNA==}
     dependencies:
       '@push.rocks/lik': 6.0.12
       '@push.rocks/smartpromise': 4.0.3

test/test.ts

@@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => {
   await testSmartarchive.exportToFs(testPaths.nogitDir);
 });
 
-tap.skip.test('should extract a b2zip', async () => {
+tap.test('should extract a b2zip', async () => {
   const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
   const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
     await testArchive.exportToFs(
@@ -47,4 +47,4 @@ tap.skip.test('should extract a b2zip', async () => {
     );
 })
 
-tap.start();
+await tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartarchive',
-  version: '4.0.13',
+  version: '4.0.17',
   description: 'work with archives'
 }

ts/bzip2/index.ts

@@ -49,13 +49,16 @@ export function unbzip2Stream() {
     } catch (e) {
       console.error(e);
       broken = true;
-      return false;
     }
   };
-
+  let counter = 0;
   return new plugins.smartstream.SmartDuplex({
+    objectMode: true,
+    name: 'bzip2',
+    debug: true,
+    highWaterMark: 1,
     writeFunction: async function (data, streamTools) {
-      //console.error('received', data.length,'bytes in', typeof data);
+      // console.log(`got chunk ${counter++}`)
       bufferQueue.push(data);
       hasBytes += data.length;
       if (bitReader === null) {
@@ -66,6 +69,10 @@ export function unbzip2Stream() {
       while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
         //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
         const result = await decompressAndPush();
+        if (!result) {
+          continue;
+        }
+        // console.log(result.toString());
         await streamTools.push(result);
       }
     },
@@ -73,11 +80,10 @@ export function unbzip2Stream() {
       //console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
       while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
         const result = await decompressAndPush();
-        streamTools.push(result);
+        await streamTools.push(result);
       }
       if (!broken) {
         if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
-        this.queue(null);
       }
     },
   });

ts/classes.archiveanalyzer.ts

@@ -4,7 +4,7 @@ import * as plugins from './plugins.js';
 export interface IAnalyzedResult {
   fileType: plugins.fileType.FileTypeResult;
   isArchive: boolean;
-  resultStream: plugins.smartstream.PassThrough;
+  resultStream: plugins.smartstream.SmartDuplex;
   decompressionStream: plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract;
 }
 
@@ -42,31 +42,30 @@ export class ArchiveAnalyzer {
         return this.smartArchiveRef.tarTools.getDecompressionStream(); // replace with your own tar decompression stream
       default:
         // Handle unsupported formats or no decompression needed
-        return new plugins.smartstream.PassThrough();
+        return plugins.smartstream.createPassThrough();
     }
   }
 
   public getAnalyzedStream() {
     let firstRun = true;
-    const resultStream = new plugins.smartstream.PassThrough();
+    const resultStream = plugins.smartstream.createPassThrough();
     const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
       readableObjectMode: true,
       writeFunction: async (chunkArg: Buffer, streamtools) => {
-        const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
-        const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
-        resultStream.push(chunkArg);
         if (firstRun) {
           firstRun = false;
+          const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
+          const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
           const result: IAnalyzedResult = {
             fileType,
             isArchive: await this.mimeTypeIsArchive(fileType?.mime),
             resultStream,
             decompressionStream,
           };
-          streamtools.push(result);
-          streamtools.push(null);
-          return null;
+          await streamtools.push(result);
         }
+        await resultStream.backpressuredPush(chunkArg);
+        return null;
       },
       finalFunction: async (tools) => {
         resultStream.push(null);