4.0.6

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartarchive",
-  "version": "4.0.0",
+  "version": "4.0.6",
   "description": "work with archives",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -26,11 +26,11 @@
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.20",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.0.7",
+    "@push.rocks/smartstream": "^3.0.12",
     "@push.rocks/smartunique": "^3.0.6",
     "@push.rocks/smarturl": "^3.0.7",
-    "@types/tar-stream": "^3.1.2",
-    "@types/unbzip2-stream": "^1.4.2",
+    "@types/tar-stream": "^3.1.3",
+    "@types/unbzip2-stream": "^1.4.3",
     "fflate": "^0.8.1",
     "file-type": "^18.6.0",
     "tar-stream": "^3.1.6",

pnpm-lock.yaml

@@ -21,8 +21,8 @@ dependencies:
     specifier: ^3.0.7
     version: 3.0.7
   '@push.rocks/smartstream':
-    specifier: ^3.0.7
-    version: 3.0.7
+    specifier: ^3.0.12
+    version: 3.0.12
   '@push.rocks/smartunique':
     specifier: ^3.0.6
     version: 3.0.6
@@ -30,11 +30,11 @@ dependencies:
     specifier: ^3.0.7
     version: 3.0.7
   '@types/tar-stream':
-    specifier: ^3.1.2
-    version: 3.1.2
+    specifier: ^3.1.3
+    version: 3.1.3
   '@types/unbzip2-stream':
-    specifier: ^1.4.2
-    version: 1.4.2
+    specifier: ^1.4.3
+    version: 1.4.3
   fflate:
     specifier: ^0.8.1
     version: 0.8.1
@@ -54,10 +54,10 @@ devDependencies:
     version: 2.1.70
   '@git.zone/tsrun':
     specifier: ^1.2.44
-    version: 1.2.46(@types/node@20.8.10)
+    version: 1.2.46(@types/node@20.9.0)
   '@git.zone/tstest':
     specifier: ^1.0.77
-    version: 1.0.81(@types/node@20.8.10)(sinon@17.0.1)
+    version: 1.0.81(@types/node@20.9.0)(sinon@17.0.1)
   '@push.rocks/tapbundle':
     specifier: ^5.0.15
     version: 5.0.15(sinon@17.0.1)
@@ -403,13 +403,13 @@ packages:
       - supports-color
     dev: true
 
-  /@git.zone/tsrun@1.2.46(@types/node@20.8.10):
+  /@git.zone/tsrun@1.2.46(@types/node@20.9.0):
     resolution: {integrity: sha512-8miFVBle9Mnjx+uPGI/P+EuWcIOXWjBAkdjN5IYbdp5Ytt4xQODCLh4JSnC9h56UeU1nUxCAxZeJs2e9TXrivA==}
     hasBin: true
     dependencies:
       '@push.rocks/smartfile': 10.0.41
       '@push.rocks/smartshell': 3.0.3
-      ts-node: 10.9.1(@types/node@20.8.10)(typescript@5.1.6)
+      ts-node: 10.9.1(@types/node@20.9.0)(typescript@5.1.6)
       typescript: 5.1.6
     transitivePeerDependencies:
       - '@swc/core'
@@ -417,13 +417,13 @@ packages:
       - '@types/node'
     dev: true
 
-  /@git.zone/tstest@1.0.81(@types/node@20.8.10)(sinon@17.0.1):
+  /@git.zone/tstest@1.0.81(@types/node@20.9.0)(sinon@17.0.1):
     resolution: {integrity: sha512-OUTACsOmvvzbKay4i1hbBaToLzo/OaCYV1PiYpoK9S2BJMbVJAik+ZbdixbJugoLrw7bR9velYgT66lnS6OZUA==}
     hasBin: true
     dependencies:
       '@api.global/typedserver': 3.0.9
       '@git.zone/tsbundle': 2.0.10
-      '@git.zone/tsrun': 1.2.46(@types/node@20.8.10)
+      '@git.zone/tsrun': 1.2.46(@types/node@20.9.0)
       '@push.rocks/consolecolor': 2.0.1
       '@push.rocks/smartbrowser': 2.0.6
       '@push.rocks/smartdelay': 3.0.5
@@ -717,7 +717,7 @@ packages:
       '@push.rocks/smartpath': 5.0.11
       '@push.rocks/smartpromise': 4.0.3
       '@push.rocks/smartrequest': 2.0.20
-      '@push.rocks/smartstream': 3.0.7
+      '@push.rocks/smartstream': 3.0.12
       '@types/fs-extra': 11.0.3
       '@types/glob': 8.1.0
       '@types/js-yaml': 4.0.8
@@ -934,8 +934,8 @@ packages:
       through2: 4.0.2
     dev: true
 
-  /@push.rocks/smartstream@3.0.7:
-    resolution: {integrity: sha512-F4HsYlMJusa7uf18aIXGuuAdlPxKaIcr7UDMLg4QUCtGK114SVt6E+72bXtN6yPyZ40+x8BVUWUkkTOdw22BeA==}
+  /@push.rocks/smartstream@3.0.12:
+    resolution: {integrity: sha512-lAlHb69JBsU47vjkV8tut6R21CKjjdBg2UFwWLKoz1U1y9j9/WR4+L4dcHm7/6AqNTVDd/hwv3Kku82o5U/39Q==}
     dependencies:
       '@push.rocks/smartpromise': 4.0.3
       '@push.rocks/smartrx': 3.0.7
@@ -1593,6 +1593,11 @@ packages:
     dependencies:
       undici-types: 5.26.5
 
+  /@types/node@20.9.0:
+    resolution: {integrity: sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==}
+    dependencies:
+      undici-types: 5.26.5
+
   /@types/parse5@6.0.3:
     resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==}
     dev: true
@@ -1655,10 +1660,10 @@ packages:
   /@types/symbol-tree@3.2.4:
     resolution: {integrity: sha512-j9qIwMi0fGQLssA0/06i8XfqCq9eKsbTu+c9uPx1T9zWisSaFWEE2CwJccn/idKCllXUeHaMSeH/i4HQ6JKEAg==}
 
-  /@types/tar-stream@3.1.2:
-    resolution: {integrity: sha512-qnIpUItVb5u8jl3kbrHofkM40ggO3YKSzc7TWqLYjDdwlrL7CiEAkDySaGfeUBLtC50RTfh2acdz51ItUbV7pQ==}
+  /@types/tar-stream@3.1.3:
+    resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}
     dependencies:
-      '@types/node': 20.8.10
+      '@types/node': 20.9.0
     dev: false
 
   /@types/through2@2.0.40:
@@ -1666,10 +1671,10 @@ packages:
     dependencies:
       '@types/node': 20.8.10
 
-  /@types/through@0.0.32:
-    resolution: {integrity: sha512-7XsfXIsjdfJM2wFDRAtEWp3zb2aVPk5QeyZxGlVK57q4u26DczMHhJmlhr0Jqv0THwxam/L8REXkj8M2I/lcvw==}
+  /@types/through@0.0.33:
+    resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==}
     dependencies:
-      '@types/node': 20.8.10
+      '@types/node': 20.9.0
     dev: false
 
   /@types/trusted-types@2.0.5:
@@ -1682,10 +1687,10 @@ packages:
       source-map: 0.6.1
     dev: true
 
-  /@types/unbzip2-stream@1.4.2:
-    resolution: {integrity: sha512-CAbJAGpAAnKArblf/GHm0M1Xs2cK3ulxYWlisuPiaSNqjCC8bvNjIsveHOft38GFXbF2vDktxdB8AW/RuNzPXA==}
+  /@types/unbzip2-stream@1.4.3:
+    resolution: {integrity: sha512-D8X5uuJRISqc8YtwL8jNW2FpPdUOCYXbfD6zNROCTbVXK9nawucxh10tVXE3MPjnHdRA1LvB0zDxVya/lBsnYw==}
     dependencies:
-      '@types/through': 0.0.32
+      '@types/through': 0.0.33
     dev: false
 
   /@types/uuid@7.0.7:
@@ -1709,11 +1714,11 @@ packages:
       '@types/node': 20.8.10
     dev: true
 
-  /@types/yauzl@2.10.2:
-    resolution: {integrity: sha512-Km7XAtUIduROw7QPgvcft0lIupeG8a8rdKL8RiSyKvlE7dYY31fEn41HVuQsRFDuROA8tA4K2UVL+WdfFmErBA==}
+  /@types/yauzl@2.10.3:
+    resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==}
     requiresBuild: true
     dependencies:
-      '@types/node': 20.8.10
+      '@types/node': 20.9.0
     dev: true
     optional: true
 
@@ -2841,7 +2846,7 @@ packages:
       get-stream: 5.2.0
       yauzl: 2.10.0
     optionalDependencies:
-      '@types/yauzl': 2.10.2
+      '@types/yauzl': 2.10.3
     transitivePeerDependencies:
       - supports-color
     dev: true
@@ -5213,7 +5218,7 @@ packages:
     hasBin: true
     dev: true
 
-  /ts-node@10.9.1(@types/node@20.8.10)(typescript@5.1.6):
+  /ts-node@10.9.1(@types/node@20.9.0)(typescript@5.1.6):
     resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==}
     hasBin: true
     peerDependencies:
@@ -5232,7 +5237,7 @@ packages:
       '@tsconfig/node12': 1.0.11
       '@tsconfig/node14': 1.0.3
       '@tsconfig/node16': 1.0.4
-      '@types/node': 20.8.10
+      '@types/node': 20.9.0
       acorn: 8.11.2
       acorn-walk: 8.3.0
       arg: 4.1.3

test/test.ts

@@ -35,21 +35,16 @@ tap.test('should extract existing files on disk', async () => {
   const testSmartarchive = await smartarchive.SmartArchive.fromArchiveUrl(
     'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
   );
-  const streamfileStream = await testSmartarchive.exportToStreamOfStreamFiles();
-  
-  streamfileStream.pipe(new plugins.smartstream.SmartDuplex({
-    objectMode: true,
-    writeAndTransformFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
-      console.log(chunkArg.relativeFilePath);
-      const streamFile = chunkArg;
-      const readStream = await streamFile.createReadStream();
-      const writePath = plugins.path.join(testPaths.nogitDir + streamFile.relativeFilePath);
-      const dir = plugins.path.parse(writePath).dir;
-      await plugins.smartfile.fs.ensureDir(plugins.path.dirname(dir));
-      const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
-      readStream.pipe(writeStream);
-    },
-  }));
+  await testSmartarchive.exportToFs(testPaths.nogitDir);
 });
 
+tap.skip.test('should extract a b2zip', async () => {
+  const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
+  const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
+    await testArchive.exportToFs(
+      plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'),
+      'data.jsonl',
+    );
+})
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartarchive',
-  version: '4.0.0',
+  version: '4.0.6',
   description: 'work with archives'
 }

ts/classes.archiveanalyzer.ts

@@ -51,15 +51,15 @@ export class ArchiveAnalyzer {
     const resultStream = new plugins.smartstream.PassThrough();
     const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
       readableObjectMode: true,
-      writeAndTransformFunction: async (chunkArg: Buffer, streamtools) => {
+      writeFunction: async (chunkArg: Buffer, streamtools) => {
         const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
-        const decompressionStream = this.getDecompressionStream(fileType.mime as any);
+        const decompressionStream = this.getDecompressionStream(fileType?.mime as any);
         resultStream.push(chunkArg);
         if (firstRun) {
           firstRun = false;
           const result: IAnalyzedResult = {
             fileType,
-            isArchive: await this.mimeTypeIsArchive(fileType.mime),
+            isArchive: await this.mimeTypeIsArchive(fileType?.mime),
             resultStream,
             decompressionStream,
           };
@@ -68,6 +68,10 @@ export class ArchiveAnalyzer {
           return null;
         }
       },
+      finalFunction: async (tools) => {
+        resultStream.push(null);
+        return null;
+      }
     });
     return analyzerstream;
   }

ts/classes.smartarchive.ts

@@ -74,7 +74,27 @@ export class SmartArchive {
     // return archiveStream;
   }
 
-  public async exportToFs(targetDir: string): Promise<void> {}
+  public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
+    const done = plugins.smartpromise.defer<void>();
+    const streamFileStream = await this.exportToStreamOfStreamFiles();
+    streamFileStream.pipe(new plugins.smartstream.SmartDuplex({
+      objectMode: true,
+      writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
+        console.log(chunkArg.relativeFilePath);
+        const streamFile = chunkArg;
+        const readStream = await streamFile.createReadStream();
+        await plugins.smartfile.fs.ensureDir(targetDir);
+        const writePath = plugins.path.join(targetDir, (streamFile.relativeFilePath || fileNameArg));
+        await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
+        const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
+        readStream.pipe(writeStream);
+      },
+      finalFunction: async () => {
+        done.resolve();
+      }
+    }));
+    return done.promise;
+  }
 
   public async exportToStreamOfStreamFiles() {
     const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
@@ -87,8 +107,9 @@ export class SmartArchive {
     const createUnpackStream = () =>
       plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
         async (analyzedResultChunk) => {
-          if (analyzedResultChunk.fileType.mime === 'application/x-tar') {
-            (analyzedResultChunk.decompressionStream as plugins.tarStream.Extract).on(
+          if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
+            const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
+            tarStream.on(
               'entry',
               async (header, stream, next) => {
                 const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
@@ -96,8 +117,13 @@ export class SmartArchive {
                 stream.on('end', function () {
                   next(); // ready for next entry
                 });
+                stream.resume(); // just auto drain the stream
               }
             );
+            tarStream.on('finish', function () {
+              console.log('finished');
+              streamFileIntake.signalEnd();
+            });
             analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
           } else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
             analyzedResultChunk.resultStream
@@ -107,11 +133,12 @@ export class SmartArchive {
           } else {
             const streamFile = plugins.smartfile.StreamFile.fromStream(
               analyzedResultChunk.resultStream,
-              analyzedResultChunk.fileType.ext
+              analyzedResultChunk.fileType?.ext
             );
             streamFileIntake.push(streamFile);
+            streamFileIntake.signalEnd();
           }
-        }
+        },
       );
 
     archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());