2.0.4

.gitignore

@@ -15,8 +15,6 @@ node_modules/
 
 # builds
 dist/
-dist_web/
+dist_*/
-dist_serve/
-dist_ts_web/
 
 # custom

.gitlab-ci.yml

@@ -19,22 +19,35 @@ mirror:
   stage: security
   script:
     - npmci git mirror
+  only:
+    - tags
   tags:
     - lossless
     - docker
     - notpriv
 
-snyk:
+auditProductionDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:snyk
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  stage: security
+  script:
+    - npmci npm prepare
+    - npmci command npm install --production --ignore-scripts
+    - npmci command npm config set registry https://registry.npmjs.org
+    - npmci command npm audit --audit-level=high --only=prod --production
+  tags:
+    - docker
+
+auditDevDependencies:
+  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
   stage: security
   script:
     - npmci npm prepare
     - npmci command npm install --ignore-scripts
-    - npmci command snyk test
+    - npmci command npm config set registry https://registry.npmjs.org
+    - npmci command npm audit --audit-level=high --only=dev
   tags:
-    - lossless
     - docker
-    - notpriv
+  allow_failure: true
 
 # ====================
 # test stage
@@ -49,9 +62,7 @@ testStable:
     - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - priv
 
 testBuild:
   stage: test
@@ -62,9 +73,7 @@ testBuild:
     - npmci command npm run build
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
-    - lossless
     - docker
-    - notpriv
 
 release:
   stage: release
@@ -84,6 +93,8 @@ release:
 codequality:
   stage: metadata
   allow_failure: true
+  only:
+    - tags
   script:
     - npmci command npm install -g tslint typescript
     - npmci npm prepare

.vscode/settings.json

@@ -15,7 +15,7 @@
             "properties": {
               "projectType": {
                 "type": "string",
-                "enum": ["website", "element", "service", "npm"]
+                "enum": ["website", "element", "service", "npm", "wcc"]
               }
             }
           }

npmextra.json

@@ -1,6 +1,7 @@
 {
   "npmci": {
-    "npmGlobalTools": []
+    "npmGlobalTools": [],
+    "npmAccessLevel": "public"
   },
   "gitzone": {
     "projectType": "npm",

package.json

@@ -1,12 +1,12 @@
 {
-  "name": "smartarchive",
+  "name": "@pushrocks/smartarchive",
-  "version": "1.0.6",
+  "version": "2.0.4",
   "description": "work with archives",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
   "scripts": {
-    "test": "tstest test/",
+    "test": "(tstest test/ --web)",
-    "build": "tsbuild"
+    "build": "tsbuild  --web"
   },
   "repository": {
     "type": "git",
@@ -19,18 +19,24 @@
   },
   "homepage": "https://github.com/pushrocks/smartarchive#readme",
   "dependencies": {
-    "@pushrocks/smartfile": "^7.0.10",
+    "@pushrocks/smartfile": "^8.0.9",
-    "@pushrocks/smartpath": "^4.0.1",
+    "@pushrocks/smartpath": "^4.0.3",
-    "@pushrocks/smartrequest": "^1.1.47",
+    "@pushrocks/smartrequest": "^1.1.51",
+    "@pushrocks/smartrx": "^2.0.19",
     "@pushrocks/smartunique": "^3.0.3",
-    "@types/tar": "^4.0.3",
+    "@pushrocks/streamfunction": "^2.0.1",
-    "tar": "^6.0.1"
+    "@types/gunzip-maybe": "^1.4.0",
+    "@types/tar": "^4.0.4",
+    "@types/tar-stream": "^2.2.0",
+    "gunzip-maybe": "^1.4.2",
+    "tar": "^6.1.0",
+    "tar-stream": "^2.2.0"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.22",
+    "@gitzone/tsbuild": "^2.1.25",
-    "@gitzone/tstest": "^1.0.28",
+    "@gitzone/tstest": "^1.0.52",
-    "@pushrocks/tapbundle": "^3.2.1",
+    "@pushrocks/tapbundle": "^3.2.14",
-    "tslint": "^6.1.0",
+    "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0"
   },
   "private": false,
@@ -38,11 +44,15 @@
     "ts/**/*",
     "ts_web/**/*",
     "dist/**/*",
-    "dist_web/**/*",
+    "dist_*/**/*",
+    "dist_ts/**/*",
     "dist_ts_web/**/*",
     "assets/**/*",
     "cli.js",
     "npmextra.json",
     "readme.md"
+  ],
+  "browserslist": [
+    "last 1 chrome versions"
   ]
 }

readme.md

@@ -8,13 +8,20 @@ work with archives
 * [docs (typedoc)](https://pushrocks.gitlab.io/smartarchive/)
 
 ## Status for master
-[![pipeline status](https://gitlab.com/pushrocks/smartarchive/badges/master/pipeline.svg)](https://gitlab.com/pushrocks/smartarchive/commits/master)
+
-[![coverage report](https://gitlab.com/pushrocks/smartarchive/badges/master/coverage.svg)](https://gitlab.com/pushrocks/smartarchive/commits/master)
+Status Category | Status Badge
-[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/smartarchive.svg)](https://www.npmjs.com/package/@pushrocks/smartarchive)
+-- | --
-[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/smartarchive/badge.svg)](https://snyk.io/test/npm/@pushrocks/smartarchive)
+GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartarchive/badges/master/pipeline.svg)](https://lossless.cloud)
-[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartarchive/badges/master/coverage.svg)](https://lossless.cloud)
-[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartarchive)](https://lossless.cloud)
-[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
+Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartarchive)](https://lossless.cloud)
+TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
+node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
+Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
+PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartarchive)](https://lossless.cloud)
+PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartarchive)](https://lossless.cloud)
+BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartarchive)](https://lossless.cloud)
+Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
 
 ## Usage
 
@@ -25,7 +32,7 @@ import * as smartarchive from 'smartarchive';
 smartarchive
   .get({
     from: 'https://example.com/example.zip',
-    toPath: '/some/local/absolute/path'
+    toPath: '/some/local/absolute/path',
   })
   .then(/*...*/);
 ```
@@ -37,7 +44,6 @@ For further information read the linked docs at the top of this README.
 
 [![repo-footer](https://pushrocks.gitlab.io/assets/repo-footer.svg)](https://push.rocks)
 
-
 ## Contribution
 
 We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)

test/test.ts

@@ -7,32 +7,89 @@ import * as smartrequest from '@pushrocks/smartrequest';
 const testPlugins = {
   path,
   smartfile,
-  smartrequest
+  smartrequest,
 };
 
 const testPaths = {
-  nogitDir: testPlugins.path.join(process.cwd(), '.nogit/')
+  nogitDir: testPlugins.path.join(__dirname, '../.nogit/'),
-}
+  remoteDir: testPlugins.path.join(__dirname, '../.nogit/remote'),
+};
 
 import * as smartarchive from '../ts/index';
 
 tap.preTask('should prepare .nogit dir', async () => {
-  await testPlugins.smartfile.fs.ensureDir(testPaths.nogitDir);
+  await testPlugins.smartfile.fs.ensureDir(testPaths.remoteDir);
 });
 
-tap.preTask('should prepare downloads', async tools => {
+tap.preTask('should prepare downloads', async (tools) => {
   const downloadedFile: Buffer = (
-    await testPlugins.smartrequest.getBinary('https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
+    await testPlugins.smartrequest.getBinary(
+      'https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
+    )
   ).body;
-  await testPlugins.smartfile.memory.toFs(downloadedFile, testPlugins.path.join(testPaths.nogitDir, 'test.tgz'));
+  await testPlugins.smartfile.memory.toFs(
+    downloadedFile,
+    testPlugins.path.join(testPaths.nogitDir, 'test.tgz')
+  );
 });
 
-tap.test('should extract files on disk', async () => {
+tap.test('should extract existing files on disk', async () => {
   const testSmartarchive = new smartarchive.SmartArchive();
-  await testSmartarchive.extractArchiveFromFilePath(
+  await testSmartarchive.extractArchiveFromFilePathToFs(
     testPlugins.path.join(testPaths.nogitDir, 'test.tgz'),
     testPlugins.path.join(testPaths.nogitDir)
   );
 });
 
+tap.test('should download a package from the registry', async () => {
+  const testSmartarchive = new smartarchive.SmartArchive();
+  await testSmartarchive.extractArchiveFromUrlToFs(
+    'https://verdaccio.lossless.one/@pushrocks%2fsmartfile/-/smartfile-7.0.11.tgz',
+    testPaths.remoteDir
+  );
+});
+
+tap.test('should extract a package using tarStream', async (tools) => {
+  const done = tools.defer();
+  const testSmartarchive = new smartarchive.SmartArchive();
+  const testTgzBuffer = (
+    await testPlugins.smartfile.Smartfile.fromFilePath(
+      testPlugins.path.join(testPaths.nogitDir, 'test.tgz')
+    )
+  ).contentBuffer;
+  const extractionFileObservable = await testSmartarchive.extractArchiveFromBufferToObservable(
+    testTgzBuffer
+  );
+  const subscription = extractionFileObservable.subscribe(
+    (file) => {
+      console.log(file.path);
+    },
+    (err) => {
+      console.log(err);
+    },
+    () => {
+      done.resolve();
+    }
+  );
+  await done.promise;
+});
+
+tap.test('should extract a file from url to replaySubject', async (tools) => {
+  const done = tools.defer();
+  const testSmartarchive = new smartarchive.SmartArchive();
+  const extractionFileObservable = await testSmartarchive.extractArchiveFromUrlToObservable(
+    'https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
+  );
+  const subscription = extractionFileObservable.subscribe((file) => {
+    console.log(file.path);
+  },
+  (err) => {
+    console.log(err);
+  },
+  () => {
+    done.resolve();
+  });
+  await done.promise;
+});
+
 tap.start();

ts/smartarchive.classes.smartarchive.ts

@@ -1,5 +1,6 @@
 import * as plugins from './smartarchive.plugins';
 import * as paths from './smartarchive.paths';
+import { extract } from 'tar';
 
 export class SmartArchive {
   public archiveDirectory: string;
@@ -8,17 +9,15 @@ export class SmartArchive {
   /**
    * extracts an archive from a given url
    */
-  public async extractArchiveFromUrl(urlArg: string, targetDir: string) {
+  public async extractArchiveFromUrlToFs(urlArg: string, targetDir: string) {
     const parsedPath = plugins.path.parse(urlArg);
     const uniqueFileName = plugins.smartunique.uni() + parsedPath.ext;
+    plugins.smartfile.fs.ensureDir(paths.nogitDir); // TODO: totally remove caching needs
     const downloadPath = plugins.path.join(paths.nogitDir, uniqueFileName);
-    const downloadedArchive = (
+    const downloadedArchive = (await plugins.smartrequest.getBinary(urlArg)).body;
-      await plugins.smartrequest.getBinary(
-        'https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
-      )
-    ).body;
     await plugins.smartfile.memory.toFs(downloadedArchive, downloadPath);
-    await this.extractArchiveFromFilePath(downloadPath, targetDir);
+    await this.extractArchiveFromFilePathToFs(downloadPath, targetDir);
+    await plugins.smartfile.fs.remove(downloadPath);
   }
 
   /**
@@ -26,16 +25,69 @@ export class SmartArchive {
    * @param filePathArg
    * @param targetDir
    */
-  public async extractArchiveFromFilePath(filePathArg: string, targetDir: string) {
+  public async extractArchiveFromFilePathToFs(filePathArg: string, targetDir: string) {
     const parsedPath = plugins.path.parse(filePathArg);
     switch (parsedPath.ext) {
       case '.tgz':
         console.log(`detected a .tgz archive`);
         await plugins.tar.extract({
           file: filePathArg,
-          cwd: targetDir
+          cwd: targetDir,
         });
         break;
     }
   }
+
+  /**
+   * extracts to Observable
+   */
+  public async extractArchiveFromBufferToObservable(
+    bufferArg: Buffer
+  ): Promise<plugins.smartrx.rxjs.ReplaySubject<plugins.smartfile.Smartfile>> {
+    const intake = new plugins.streamfunction.Intake();
+    const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<plugins.smartfile.Smartfile>();
+    const readableStream = intake.getReadableStream();
+    const extractPipeStop = plugins.tarStream.extract();
+    extractPipeStop.on('entry', (header, stream, next) => {
+      let fileBuffer: Buffer;
+      stream.on('data', (chunkArg) => {
+        if (!fileBuffer) {
+          fileBuffer = chunkArg;
+        } else {
+          fileBuffer = Buffer.concat([fileBuffer, chunkArg]);
+        }
+      });
+      stream.on('end', () => {
+        replaySubject.next(
+          new plugins.smartfile.Smartfile({
+            contentBuffer: fileBuffer,
+            path: header.name,
+          })
+        );
+        next();
+      });
+      stream.resume();
+    });
+    extractPipeStop.on('finish', () => {
+      replaySubject.complete();
+    });
+    // lets run the stream
+    readableStream
+      .pipe(plugins.gunzipMaybe())
+      .pipe(extractPipeStop);
+    intake.pushData(bufferArg);
+    intake.signalEnd();
+    return replaySubject;
+  }
+
+  /**
+   * extracts to Observable
+   */
+   public async extractArchiveFromUrlToObservable(
+    urlArg: string
+  ): Promise<plugins.smartrx.rxjs.ReplaySubject<plugins.smartfile.Smartfile>> {
+    const response = await plugins.smartrequest.getBinary(urlArg);
+    const replaySubject = this.extractArchiveFromBufferToObservable(response.body);
+    return replaySubject;
+  }
 }

ts/smartarchive.plugins.ts

@@ -8,10 +8,14 @@ import * as smartfile from '@pushrocks/smartfile';
 import * as smartpath from '@pushrocks/smartpath';
 import * as smartrequest from '@pushrocks/smartrequest';
 import * as smartunique from '@pushrocks/smartunique';
+import * as streamfunction from '@pushrocks/streamfunction';
+import * as smartrx from '@pushrocks/smartrx';
 
-export { smartfile, smartpath, smartrequest, smartunique };
+export { smartfile, smartpath, smartrequest, smartunique, streamfunction, smartrx };
 
 // third party scope
+import gunzipMaybe from 'gunzip-maybe';
 import tar from 'tar';
+import tarStream from 'tar-stream';
 
-export { tar };
+export { gunzipMaybe, tar, tarStream };