4.0.5

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartarchive",
-  "version": "4.0.0",
+  "version": "4.0.5",
   "description": "work with archives",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -26,7 +26,7 @@
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrequest": "^2.0.20",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.0.7",
+    "@push.rocks/smartstream": "^3.0.11",
     "@push.rocks/smartunique": "^3.0.6",
     "@push.rocks/smarturl": "^3.0.7",
     "@types/tar-stream": "^3.1.2",

pnpm-lock.yaml

@@ -21,8 +21,8 @@ dependencies:
     specifier: ^3.0.7
     version: 3.0.7
   '@push.rocks/smartstream':
-    specifier: ^3.0.7
-    version: 3.0.7
+    specifier: ^3.0.11
+    version: 3.0.11
   '@push.rocks/smartunique':
     specifier: ^3.0.6
     version: 3.0.6
@@ -717,7 +717,7 @@ packages:
       '@push.rocks/smartpath': 5.0.11
       '@push.rocks/smartpromise': 4.0.3
       '@push.rocks/smartrequest': 2.0.20
-      '@push.rocks/smartstream': 3.0.7
+      '@push.rocks/smartstream': 3.0.11
       '@types/fs-extra': 11.0.3
       '@types/glob': 8.1.0
       '@types/js-yaml': 4.0.8
@@ -934,8 +934,8 @@ packages:
       through2: 4.0.2
     dev: true
 
-  /@push.rocks/smartstream@3.0.7:
-    resolution: {integrity: sha512-F4HsYlMJusa7uf18aIXGuuAdlPxKaIcr7UDMLg4QUCtGK114SVt6E+72bXtN6yPyZ40+x8BVUWUkkTOdw22BeA==}
+  /@push.rocks/smartstream@3.0.11:
+    resolution: {integrity: sha512-MrJGCXcUYliAZlE/ozGzCj6Udtg/2f4OfJCd/7We8tK2kS+YWl+TSvubz8KFbUqcl5dqHQro0txrmVcthd9gEQ==}
     dependencies:
       '@push.rocks/smartpromise': 4.0.3
       '@push.rocks/smartrx': 3.0.7

test/test.ts

@@ -35,21 +35,16 @@ tap.test('should extract existing files on disk', async () => {
   const testSmartarchive = await smartarchive.SmartArchive.fromArchiveUrl(
     'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
   );
-  const streamfileStream = await testSmartarchive.exportToStreamOfStreamFiles();
-  
-  streamfileStream.pipe(new plugins.smartstream.SmartDuplex({
-    objectMode: true,
-    writeAndTransformFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
-      console.log(chunkArg.relativeFilePath);
-      const streamFile = chunkArg;
-      const readStream = await streamFile.createReadStream();
-      const writePath = plugins.path.join(testPaths.nogitDir + streamFile.relativeFilePath);
-      const dir = plugins.path.parse(writePath).dir;
-      await plugins.smartfile.fs.ensureDir(plugins.path.dirname(dir));
-      const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
-      readStream.pipe(writeStream);
-    },
-  }));
+  await testSmartarchive.exportToFs(testPaths.nogitDir);
 });
 
+tap.skip.test('should extract a b2zip', async () => {
+  const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
+  const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
+    await testArchive.exportToFs(
+      plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'),
+      'data.jsonl',
+    );
+})
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartarchive',
-  version: '4.0.0',
+  version: '4.0.5',
   description: 'work with archives'
 }

ts/classes.archiveanalyzer.ts

@@ -53,13 +53,13 @@ export class ArchiveAnalyzer {
       readableObjectMode: true,
       writeAndTransformFunction: async (chunkArg: Buffer, streamtools) => {
         const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
-        const decompressionStream = this.getDecompressionStream(fileType.mime as any);
+        const decompressionStream = this.getDecompressionStream(fileType?.mime as any);
         resultStream.push(chunkArg);
         if (firstRun) {
           firstRun = false;
           const result: IAnalyzedResult = {
             fileType,
-            isArchive: await this.mimeTypeIsArchive(fileType.mime),
+            isArchive: await this.mimeTypeIsArchive(fileType?.mime),
             resultStream,
             decompressionStream,
           };
@@ -68,6 +68,10 @@ export class ArchiveAnalyzer {
           return null;
         }
       },
+      finalFunction: async (tools) => {
+        resultStream.push(null);
+        return null;
+      }
     });
     return analyzerstream;
   }

ts/classes.smartarchive.ts

@@ -74,7 +74,27 @@ export class SmartArchive {
     // return archiveStream;
   }
 
-  public async exportToFs(targetDir: string): Promise<void> {}
+  public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
+    const done = plugins.smartpromise.defer<void>();
+    const streamFileStream = await this.exportToStreamOfStreamFiles();
+    streamFileStream.pipe(new plugins.smartstream.SmartDuplex({
+      objectMode: true,
+      writeAndTransformFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
+        console.log(chunkArg.relativeFilePath);
+        const streamFile = chunkArg;
+        const readStream = await streamFile.createReadStream();
+        await plugins.smartfile.fs.ensureDir(targetDir);
+        const writePath = plugins.path.join(targetDir, (streamFile.relativeFilePath || fileNameArg));
+        await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
+        const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
+        readStream.pipe(writeStream);
+      },
+      finalFunction: async () => {
+        done.resolve();
+      }
+    }));
+    return done.promise;
+  }
 
   public async exportToStreamOfStreamFiles() {
     const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
@@ -87,8 +107,9 @@ export class SmartArchive {
     const createUnpackStream = () =>
       plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
         async (analyzedResultChunk) => {
-          if (analyzedResultChunk.fileType.mime === 'application/x-tar') {
-            (analyzedResultChunk.decompressionStream as plugins.tarStream.Extract).on(
+          if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
+            const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
+            tarStream.on(
               'entry',
               async (header, stream, next) => {
                 const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
@@ -96,8 +117,13 @@ export class SmartArchive {
                 stream.on('end', function () {
                   next(); // ready for next entry
                 });
+                stream.resume(); // just auto drain the stream
               }
             );
+            tarStream.on('finish', function () {
+              console.log('finished');
+              streamFileIntake.signalEnd();
+            });
             analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
           } else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
             analyzedResultChunk.resultStream
@@ -107,11 +133,12 @@ export class SmartArchive {
           } else {
             const streamFile = plugins.smartfile.StreamFile.fromStream(
               analyzedResultChunk.resultStream,
-              analyzedResultChunk.fileType.ext
+              analyzedResultChunk.fileType?.ext
             );
             streamFileIntake.push(streamFile);
+            streamFileIntake.signalEnd();
           }
-        }
+        },
       );
 
     archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());