4.0.21

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartarchive",
-  "version": "4.0.13",
+  "version": "4.0.21",
   "description": "work with archives",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -21,25 +21,26 @@
   },
   "homepage": "https://github.com/pushrocks/smartarchive#readme",
   "dependencies": {
-    "@push.rocks/smartfile": "^11.0.0",
+    "@push.rocks/smartdelay": "^3.0.5",
+    "@push.rocks/smartfile": "^11.0.4",
     "@push.rocks/smartpath": "^5.0.11",
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.21",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.0.25",
-    "@push.rocks/smartunique": "^3.0.6",
+    "@push.rocks/smartstream": "^3.0.34",
+    "@push.rocks/smartunique": "^3.0.8",
     "@push.rocks/smarturl": "^3.0.7",
     "@types/tar-stream": "^3.1.3",
-    "fflate": "^0.8.1",
-    "file-type": "^18.7.0",
-    "tar-stream": "^3.1.6",
+    "fflate": "^0.8.2",
+    "file-type": "^19.0.0",
+    "tar-stream": "^3.1.7",
     "through": "^2.3.8"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.66",
+    "@git.zone/tsbuild": "^2.1.72",
     "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.84",
-    "@push.rocks/tapbundle": "^5.0.15"
+    "@git.zone/tstest": "^1.0.88",
+    "@push.rocks/tapbundle": "^5.0.17"
   },
   "private": false,
   "files": [

test/test.ts

@@ -47,4 +47,4 @@ tap.skip.test('should extract a b2zip', async () => {
     );
 })
 
-tap.start();
+await tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartarchive',
-  version: '4.0.13',
+  version: '4.0.21',
   description: 'work with archives'
 }

ts/bzip2/index.ts

@@ -49,13 +49,16 @@ export function unbzip2Stream() {
     } catch (e) {
       console.error(e);
       broken = true;
-      return false;
     }
   };
-
+  let counter = 0;
   return new plugins.smartstream.SmartDuplex({
+    objectMode: true,
+    name: 'bzip2',
+    debug: false,
+    highWaterMark: 1,
     writeFunction: async function (data, streamTools) {
-      //console.error('received', data.length,'bytes in', typeof data);
+      // console.log(`got chunk ${counter++}`)
       bufferQueue.push(data);
       hasBytes += data.length;
       if (bitReader === null) {
@@ -66,6 +69,10 @@ export function unbzip2Stream() {
       while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
         //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
         const result = await decompressAndPush();
+        if (!result) {
+          continue;
+        }
+        // console.log(result.toString());
         await streamTools.push(result);
       }
     },
@@ -73,11 +80,13 @@ export function unbzip2Stream() {
       //console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
       while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
         const result = await decompressAndPush();
-        streamTools.push(result);
+        if (!result) {
+          continue;
+        }
+        await streamTools.push(result);
       }
       if (!broken) {
         if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
-        this.queue(null);
       }
     },
   });

ts/classes.archiveanalyzer.ts

@@ -4,7 +4,7 @@ import * as plugins from './plugins.js';
 export interface IAnalyzedResult {
   fileType: plugins.fileType.FileTypeResult;
   isArchive: boolean;
-  resultStream: plugins.smartstream.PassThrough;
+  resultStream: plugins.smartstream.SmartDuplex;
   decompressionStream: plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract;
 }
 
@@ -36,37 +36,41 @@ export class ArchiveAnalyzer {
     switch (mimeTypeArg) {
       case 'application/gzip':
         return this.smartArchiveRef.gzipTools.getDecompressionStream();
+      case 'application/zip':
+        return this.smartArchiveRef.zipTools.getDecompressionStream();
       case 'application/x-bzip2':
         return await this.smartArchiveRef.bzip2Tools.getDecompressionStream(); // replace with your own bzip2 decompression stream
       case 'application/x-tar':
         return this.smartArchiveRef.tarTools.getDecompressionStream(); // replace with your own tar decompression stream
       default:
         // Handle unsupported formats or no decompression needed
-        return new plugins.smartstream.PassThrough();
+        return plugins.smartstream.createPassThrough();
     }
   }
 
   public getAnalyzedStream() {
     let firstRun = true;
-    const resultStream = new plugins.smartstream.PassThrough();
+    const resultStream = plugins.smartstream.createPassThrough();
     const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
       readableObjectMode: true,
       writeFunction: async (chunkArg: Buffer, streamtools) => {
-        const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
-        const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
-        resultStream.push(chunkArg);
         if (firstRun) {
           firstRun = false;
+          const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
+          const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
+          /**
+           * analyzed stream emits once with this object
+           */
           const result: IAnalyzedResult = {
             fileType,
             isArchive: await this.mimeTypeIsArchive(fileType?.mime),
             resultStream,
             decompressionStream,
           };
-          streamtools.push(result);
-          streamtools.push(null);
-          return null;
+          await streamtools.push(result);
         }
+        await resultStream.backpressuredPush(chunkArg);
+        return null;
       },
       finalFunction: async (tools) => {
         resultStream.push(null);

ts/classes.smartarchive.ts

@@ -1,9 +1,10 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 
+import { Bzip2Tools } from './classes.bzip2tools.js';
 import { GzipTools } from './classes.gziptools.js';
 import { TarTools } from './classes.tartools.js';
-import { Bzip2Tools } from './classes.bzip2tools.js';
+import { ZipTools } from './classes.ziptools.js';
 
 import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
 
@@ -32,9 +33,10 @@ export class SmartArchive {
   }
 
   // INSTANCE
-  public tarTools = new TarTools(this);
   public gzipTools = new GzipTools(this);
   public bzip2Tools = new Bzip2Tools(this);
+  public tarTools = new TarTools(this);
+  public zipTools = new ZipTools(this);
   public archiveAnalyzer = new ArchiveAnalyzer(this);
 
   public sourceUrl: string;
@@ -77,27 +79,32 @@ export class SmartArchive {
   public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
     const done = plugins.smartpromise.defer<void>();
     const streamFileStream = await this.exportToStreamOfStreamFiles();
-    streamFileStream.pipe(new plugins.smartstream.SmartDuplex({
-      objectMode: true,
-      writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
-        const done = plugins.smartpromise.defer<void>();
-        console.log(chunkArg.relativeFilePath ? chunkArg.relativeFilePath : 'no relative path');
-        const streamFile = chunkArg;
-        const readStream = await streamFile.createReadStream();
-        await plugins.smartfile.fs.ensureDir(targetDir);
-        const writePath = plugins.path.join(targetDir, (streamFile.relativeFilePath || fileNameArg));
-        await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
-        const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
-        readStream.pipe(writeStream);
-        writeStream.on('finish', () => {
+    streamFileStream.pipe(
+      new plugins.smartstream.SmartDuplex({
+        objectMode: true,
+        writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
+          const done = plugins.smartpromise.defer<void>();
+          console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path');
+          const streamFile = streamFileArg;
+          const readStream = await streamFile.createReadStream();
+          await plugins.smartfile.fs.ensureDir(targetDir);
+          const writePath = plugins.path.join(
+            targetDir,
+            streamFile.relativeFilePath || fileNameArg
+          );
+          await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
+          const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
+          readStream.pipe(writeStream);
+          writeStream.on('finish', () => {
+            done.resolve();
+          });
+          await done.promise;
+        },
+        finalFunction: async () => {
           done.resolve();
-        })
-        await done.promise;
-      },
-      finalFunction: async () => {
-        done.resolve();
-      }
-    }));
+        },
+      })
+    );
     return done.promise;
   }
 
@@ -114,21 +121,27 @@ export class SmartArchive {
         async (analyzedResultChunk) => {
           if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
             const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
-            tarStream.on(
-              'entry',
-              async (header, stream, next) => {
-                const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
-                streamFileIntake.push(streamfile);
-                stream.on('end', function () {
-                  next(); // ready for next entry
-                });
-              }
-            );
+            tarStream.on('entry', async (header, stream, next) => {
+              const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
+              streamFileIntake.push(streamfile);
+              stream.on('end', function () {
+                next(); // ready for next entry
+              });
+            });
             tarStream.on('finish', function () {
               console.log('finished');
               streamFileIntake.signalEnd();
             });
             analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
+          } else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
+            analyzedResultChunk.resultStream
+              .pipe(analyzedResultChunk.decompressionStream)
+              .pipe(new plugins.smartstream.SmartDuplex({
+                objectMode: true,
+                writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
+                  streamFileIntake.push(streamFileArg);
+                },
+              }));
           } else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
             analyzedResultChunk.resultStream
               .pipe(analyzedResultChunk.decompressionStream)
@@ -142,8 +155,9 @@ export class SmartArchive {
             streamFileIntake.push(streamFile);
             streamFileIntake.signalEnd();
           }
-        }, {
-          objectMode: true
+        },
+        {
+          objectMode: true,
         }
       );
 

ts/classes.ziptools.ts

@@ -0,0 +1,77 @@
+import type { SmartArchive } from './classes.smartarchive.js';
+import * as plugins from './plugins.js';
+
+class DecompressZipTransform extends plugins.smartstream.SmartDuplex<ArrayBufferLike> {
+  private streamtools: plugins.smartstream.IStreamTools;
+  private unzipper = new plugins.fflate.Unzip(async (fileArg) => {
+    let resultBuffer: Buffer;
+    fileArg.ondata = async (dataArg, dat, final) => {
+      resultBuffer? resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)])
+        : resultBuffer = Buffer.from(dat);
+      if (final) {
+        const streamFile = plugins.smartfile.StreamFile.fromBuffer(resultBuffer);
+        streamFile.relativeFilePath = fileArg.name;
+        this.streamtools.push(streamFile);
+      }
+    }
+    fileArg.start();
+  });
+  constructor() {
+    super({
+      objectMode: true,
+      writeFunction: async (chunkArg: Buffer, streamtoolsArg) => {
+        this.streamtools? null : this.streamtools = streamtoolsArg;
+        this.unzipper.push(chunkArg, false);
+      },
+      finalFunction: async () => {
+        this.unzipper.push(Buffer.from(''), true);
+        await plugins.smartdelay.delayFor(0);
+        this.streamtools.push(null);
+      }
+    });
+    this.unzipper.register(plugins.fflate.UnzipInflate);
+  }
+}
+
+// This class wraps fflate's zip in a Node.js Transform stream for compression
+export class CompressZipTransform extends plugins.stream.Transform {
+  files: { [fileName: string]: Uint8Array };
+
+  constructor() {
+    super();
+    this.files = {};
+  }
+
+  _transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) {
+    // Simple example: storing chunks in memory before finalizing ZIP in _flush
+    this.files['file.txt'] = new Uint8Array(chunk);
+    callback();
+  }
+
+  _flush(callback: plugins.stream.TransformCallback) {
+    plugins.fflate.zip(this.files, (err, zipped) => {
+      if (err) {
+        callback(err);
+      } else {
+        this.push(Buffer.from(zipped));
+        callback();
+      }
+    });
+  }
+}
+
+export class ZipTools {
+  smartArchiveRef: SmartArchive;
+
+  constructor(smartArchiveRefArg: SmartArchive) {
+    this.smartArchiveRef = smartArchiveRefArg;
+  }
+
+  public getCompressionStream() {
+    return new CompressZipTransform();
+  }
+
+  public getDecompressionStream() {
+    return new DecompressZipTransform();
+  }
+}

ts/plugins.ts

@@ -6,6 +6,7 @@ export { path, stream };
 
 // @pushrocks scope
 import * as smartfile from '@push.rocks/smartfile';
+import * as smartdelay from '@push.rocks/smartdelay';
 import * as smartpath from '@push.rocks/smartpath';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrequest from '@push.rocks/smartrequest';
@@ -14,7 +15,7 @@ import * as smartstream from '@push.rocks/smartstream';
 import * as smartrx from '@push.rocks/smartrx';
 import * as smarturl from '@push.rocks/smarturl';
 
-export { smartfile, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl };
+export { smartfile, smartdelay, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl };
 
 // third party scope
 import * as fileType from 'file-type';