1.1.36

.gitignore

@@ -1,4 +1,22 @@
-node_modules/
+.nogit/
+
+# artifacts
 coverage/
 public/
 pages/
+
+# installs
+node_modules/
+
+# caches
+.yarn/
+.cache/
+.rpt2_cache
+
+# builds
+dist/
+dist_web/
+dist_serve/
+dist_ts_web/
+
+# custom

.gitlab-ci.yml

@@ -1,5 +1,5 @@
-# gitzone standard
+# gitzone ci_default
-image: hosttoday/ht-docker-node:npmci
+image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
 
 cache:
   paths:
@@ -26,6 +26,7 @@ mirror:
 snyk:
   stage: security
   script:
+    - npmci npm prepare
     - npmci command npm install -g snyk
     - npmci command npm install --ignore-scripts
     - npmci command snyk test
@@ -36,38 +37,29 @@ snyk:
 # ====================
 # test stage
 # ====================
-testLEGACY:
-  stage: test
-  script:
-  - npmci node install legacy
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-  allow_failure: true
 
-testLTS:
+testStable:
-  stage: test
-  script:
-  - npmci node install lts
-  - npmci npm install
-  - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - docker
-  - notpriv
-    
-testSTABLE:
   stage: test
   script:
+  - npmci npm prepare
   - npmci node install stable
   - npmci npm install
   - npmci npm test
   coverage: /\d+.?\d+?\%\s*coverage/
   tags:
   - docker
+  - priv
+
+testBuild:
+  stage: test
+  script:
+  - npmci npm prepare
+  - npmci node install stable
+  - npmci npm install
+  - npmci command npm run build
+  coverage: /\d+.?\d+?\%\s*coverage/
+  tags:
+  - docker
   - notpriv
 
 release:
@@ -86,19 +78,12 @@ release:
 # ====================
 codequality:
   stage: metadata
-  image: docker:stable
   allow_failure: true
-  services:
-    - docker:stable-dind
   script:
-    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - npmci command npm install -g tslint typescript
-    - docker run
+    - npmci npm prepare
-        --env SOURCE_CODE="$PWD"
+    - npmci npm install
-        --volume "$PWD":/code
+    - npmci command "tslint -c tslint.json ./ts/**/*.ts"
-        --volume /var/run/docker.sock:/var/run/docker.sock
-        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
-  artifacts:
-    paths: [codeclimate.json]
   tags:
   - docker
   - priv
@@ -114,12 +99,15 @@ trigger:
   - notpriv
 
 pages:
-  image: hosttoday/ht-docker-node:npmci
+  image: hosttoday/ht-docker-dbase:npmci
+  services:
+   - docker:stable-dind
   stage: metadata
   script:
-    - npmci command npm install -g typedoc typescript
+    - npmci command npm install -g @gitzone/tsdoc
+    - npmci npm prepare
     - npmci npm install
-    - npmci command typedoc --module "commonjs" --target "ES2016" --out public/ ts/
+    - npmci command tsdoc
   tags:
     - docker
     - notpriv
@@ -130,13 +118,3 @@ pages:
     paths:
     - public
   allow_failure: true
-
-windowsCompatibility:
-  image: stefanscherer/node-windows:10-build-tools
-  stage: metadata
-  script:
-  - npm install & npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-  - windows
-  allow_failure: true

README.md

@@ -1,25 +1,20 @@
-# smartrequest
+# @pushrocks/smartrequest
-
 dropin replacement for request
 
-## Availabililty
+## Availabililty and Links
-
+* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartrequest)
-[![npm](https://push.rocks/assets/repo-button-npm.svg)](https://www.npmjs.com/package/smartrequest)
+* [gitlab.com (source)](https://gitlab.com/pushrocks/smartrequest)
-[![git](https://push.rocks/assets/repo-button-git.svg)](https://GitLab.com/pushrocks/smartrequest)
+* [github.com (source mirror)](https://github.com/pushrocks/smartrequest)
-[![git](https://push.rocks/assets/repo-button-mirror.svg)](https://github.com/pushrocks/smartrequest)
+* [docs (typedoc)](https://pushrocks.gitlab.io/smartrequest/)
-[![docs](https://push.rocks/assets/repo-button-docs.svg)](https://pushrocks.gitlab.io/smartrequest/)
 
 ## Status for master
-
+[![build status](https://gitlab.com/pushrocks/smartrequest/badges/master/build.svg)](https://gitlab.com/pushrocks/smartrequest/commits/master)
-[![build status](https://GitLab.com/pushrocks/smartrequest/badges/master/build.svg)](https://GitLab.com/pushrocks/smartrequest/commits/master)
+[![coverage report](https://gitlab.com/pushrocks/smartrequest/badges/master/coverage.svg)](https://gitlab.com/pushrocks/smartrequest/commits/master)
-[![coverage report](https://GitLab.com/pushrocks/smartrequest/badges/master/coverage.svg)](https://GitLab.com/pushrocks/smartrequest/commits/master)
+[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/smartrequest.svg)](https://www.npmjs.com/package/@pushrocks/smartrequest)
-[![npm downloads per month](https://img.shields.io/npm/dm/smartrequest.svg)](https://www.npmjs.com/package/smartrequest)
+[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/smartrequest/badge.svg)](https://snyk.io/test/npm/@pushrocks/smartrequest)
-[![Dependency Status](https://david-dm.org/pushrocks/smartrequest.svg)](https://david-dm.org/pushrocks/smartrequest)
+[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-[![bitHound Dependencies](https://www.bithound.io/github/pushrocks/smartrequest/badges/dependencies.svg)](https://www.bithound.io/github/pushrocks/smartrequest/master/dependencies/npm)
+[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-[![bitHound Code](https://www.bithound.io/github/pushrocks/smartrequest/badges/code.svg)](https://www.bithound.io/github/pushrocks/smartrequest)
+[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/)
-[![TypeScript](https://img.shields.io/badge/TypeScript-2.x-blue.svg)](https://nodejs.org/dist/latest-v6.x/docs/api/)
-[![node](https://img.shields.io/badge/node->=%206.x.x-blue.svg)](https://nodejs.org/dist/latest-v6.x/docs/api/)
-[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/)
 
 ## Usage
 
@@ -66,4 +61,9 @@ smartrequest.get('https://example.com/bigfile.mp4', optionsArg, true).then(res =
 })
 ```
 
-[![npm](https://push.rocks/assets/repo-header.svg)](https://push.rocks)
+For further information read the linked docs at the top of this readme.
+
+> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
+| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+
+[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)

npmextra.json

@@ -3,9 +3,17 @@
     "coverageTreshold": 50
   },
   "npmci": {
-    "npmGlobalTools": [
+    "npmGlobalTools": [],
-      "npmts"
-    ],
     "npmAccessLevel": "public"
+  },
+  "gitzone": {
+    "module": {
+      "githost": "gitlab.com",
+      "gitscope": "pushrocks",
+      "gitrepo": "smartrequest",
+      "shortDescription": "dropin replacement for request",
+      "npmPackagename": "@pushrocks/smartrequest",
+      "license": "MIT"
+    }
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pushrocks/smartrequest",
-  "version": "1.1.13",
+  "version": "1.1.36",
   "private": false,
   "description": "dropin replacement for request",
   "main": "dist/index.js",
@@ -23,15 +23,29 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartrequest#README",
   "dependencies": {
-    "@pushrocks/smartpromise": "^2.0.5",
+    "@pushrocks/smartpromise": "^3.0.5",
-    "@types/form-data": "^2.2.1",
+    "@types/form-data": "^2.5.0",
-    "form-data": "^2.3.2"
+    "agentkeepalive": "^4.0.2",
+    "form-data": "^2.5.1"
   },
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.0.22",
+    "@gitzone/tsbuild": "^2.1.17",
-    "@gitzone/tsrun": "^1.1.12",
+    "@gitzone/tsrun": "^1.2.8",
-    "@gitzone/tstest": "^1.0.15",
+    "@gitzone/tstest": "^1.0.24",
-    "@pushrocks/tapbundle": "^3.0.5",
+    "@pushrocks/tapbundle": "^3.0.13",
-    "@types/node": "^10.5.8"
+    "@types/node": "^12.7.8",
-  }
+    "tslint": "^5.20.0",
+    "tslint-config-prettier": "^1.18.0"
+  },
+  "files": [
+    "ts/*",
+    "ts_web/*",
+    "dist/*",
+    "dist_web/*",
+    "dist_ts_web/*",
+    "assets/*",
+    "cli.js",
+    "npmextra.json",
+    "readme.md"
+  ]
 }

test/test.ts

@@ -6,6 +6,12 @@ tap.test('should request a html document over https', async () => {
   await expect(smartrequest.getJson('https://encrypted.google.com/'))
     .to.eventually.property('body')
     .be.a('string');
+  await expect(smartrequest.getJson('https://encrypted.google.com/'))
+    .to.eventually.property('body')
+    .be.a('string');
+  await expect(smartrequest.getJson('https://encrypted.google.com/'))
+    .to.eventually.property('body')
+    .be.a('string');
 });
 
 tap.test('should request a JSON document over https', async () => {
@@ -15,7 +21,7 @@ tap.test('should request a JSON document over https', async () => {
     .equal(1);
 });
 
-tap.test('should post a JSON document over http', async () => {
+tap.skip.test('should post a JSON document over http', async () => {
   await expect(smartrequest.postJson('http://md5.jsontest.com/?text=example_text'))
     .to.eventually.property('body')
     .property('md5')
@@ -23,17 +29,18 @@ tap.test('should post a JSON document over http', async () => {
 });
 
 tap.skip.test('should deal with unix socks', async () => {
-  const socketResponse = await smartrequest.request('http://unix:/var/run/docker.sock:/containers/json', {
+  const socketResponse = await smartrequest.request(
-    headers: {
+    'http://unix:/var/run/docker.sock:/containers/json',
-      "Content-Type": "application/json",
+    {
-      "Host": "docker.sock"
+      headers: {
+        'Content-Type': 'application/json',
+        Host: 'docker.sock'
+      }
     }
-  });
+  );
   console.log(socketResponse.body);
 });
 
-tap.skip.test('should correctly upload a file using formData', async () => {
+tap.skip.test('should correctly upload a file using formData', async () => {});
-  
-})
 
 tap.start();

ts/smartrequest.formdata.ts

@@ -1,25 +1,22 @@
-import * as plugins from "./smartrequest.plugins";
+import * as plugins from './smartrequest.plugins';
-import * as interfaces from "./smartrequest.interfaces";
+import * as interfaces from './smartrequest.interfaces';
-import { request } from "./smartrequest.request";
+import { request } from './smartrequest.request';
 
 /**
  * the interfae for FormFieldData
  */
 export interface IFormField {
   name: string;
-  type: "string" | "filePath" | "Buffer";
+  type: 'string' | 'filePath' | 'Buffer';
   payload: string;
 }
 
-const appendFormField = async (
+const appendFormField = async (formDataArg: plugins.formData, formDataField: IFormField) => {
-  formDataArg: plugins.formData,
+  if (formDataField.type === 'filePath') {
-  formDataField: IFormField
+    const fileData = plugins.fs.readFileSync(
-) => {
-  if (formDataField.type === "filePath") {
-    let fileData = plugins.fs.readFileSync(
       plugins.path.join(process.cwd(), formDataField.payload)
     );
-    formDataArg.append("file", fileData, {
+    formDataArg.append('file', fileData, {
       filename: 'upload.pdf',
       contentType: 'application/pdf'
     });
@@ -35,14 +32,15 @@ export const postFormData = async (
   for (const formField of payloadArg) {
     await appendFormField(form, formField);
   }
-  const requestOptions = Object.assign({}, optionsArg, {
+  const requestOptions = {
+    ...optionsArg,
     method: 'POST',
     headers: {
-      ...(optionsArg.headers),
+      ...optionsArg.headers,
       ...form.getHeaders()
     },
     requestBody: form
-  });
+  };
 
   // lets fire the actual request for sending the formdata
   const response = await request(urlArg, requestOptions);

ts/smartrequest.interfaces.ts

@@ -2,5 +2,7 @@ import * as plugins from './smartrequest.plugins';
 import * as https from 'https';
 
 export interface ISmartRequestOptions extends https.RequestOptions {
+  keepAlive?: boolean;
   requestBody?: any;
+  autoJsonParse?: boolean;
 }

ts/smartrequest.plugins.ts

@@ -1,4 +1,4 @@
-import * as formData from 'form-data';
+import formData from 'form-data';
 import * as fs from 'fs';
 import * as http from 'http';
 import * as https from 'https';
@@ -7,4 +7,9 @@ import * as url from 'url';
 
 import * as smartpromise from '@pushrocks/smartpromise';
 
-export {formData, http, https, fs, path, url, smartpromise };
+export { formData, http, https, fs, path, url, smartpromise };
+
+// third party scope
+import * as agentkeepalive from 'agentkeepalive';
+
+export { agentkeepalive };

ts/smartrequest.request.ts

@@ -1,27 +1,31 @@
-import * as https from "https";
+import * as plugins from './smartrequest.plugins';
-import * as plugins from "./smartrequest.plugins";
+import * as interfaces from './smartrequest.interfaces';
-import * as interfaces from "./smartrequest.interfaces";
 
-import { IncomingMessage } from "http";
+import { IncomingMessage } from 'http';
 
 export interface IExtendedIncomingMessage extends IncomingMessage {
   body: any;
 }
 
 const buildUtf8Response = (
-  incomingMessageArg: IncomingMessage
+  incomingMessageArg: IncomingMessage,
+  autoJsonParse = true
 ): Promise<IExtendedIncomingMessage> => {
-  let done = plugins.smartpromise.defer<IExtendedIncomingMessage>();
+  const done = plugins.smartpromise.defer<IExtendedIncomingMessage>();
   // Continuously update stream with data
-  let body = "";
+  let body = '';
-  incomingMessageArg.on("data", function(chunkArg) {
+  incomingMessageArg.on('data', chunkArg => {
     body += chunkArg;
   });
 
-  incomingMessageArg.on("end", function() {
+  incomingMessageArg.on('end', () => {
-    try {
+    if (autoJsonParse) {
-      (incomingMessageArg as IExtendedIncomingMessage).body = JSON.parse(body);
+      try {
-    } catch (err) {
+        (incomingMessageArg as IExtendedIncomingMessage).body = JSON.parse(body);
+      } catch (err) {
+        (incomingMessageArg as IExtendedIncomingMessage).body = body;
+      }
+    } else {
       (incomingMessageArg as IExtendedIncomingMessage).body = body;
     }
     done.resolve(incomingMessageArg as IExtendedIncomingMessage);
@@ -47,79 +51,126 @@ const parseSocketPathAndRoute = (stringToParseArg: string) => {
   return {
     socketPath: result[1],
     path: result[2]
-  }
+  };
-}
+};
+
+/**
+ * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpAgent = new plugins.agentkeepalive.default();
+
+/**
+ * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpAgentKeepAliveFalse = new plugins.http.Agent({
+  maxFreeSockets: 0,
+  keepAlive: false,
+  keepAliveMsecs: 0
+});
+
+/**
+ * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpsAgent = new plugins.agentkeepalive.HttpsAgent();
+
+/**
+ * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpsAgentKeepAliveFalse = new plugins.https.Agent({
+  maxFreeSockets: 0,
+  keepAlive: false,
+  keepAliveMsecs: 0
+});
 
 export let request = async (
   domainArg: string,
   optionsArg: interfaces.ISmartRequestOptions = {},
-  streamArg: boolean = false
+  responseStreamArg: boolean = false,
+  requestDataFunc: (req: plugins.http.ClientRequest) => void = null
 ): Promise<IExtendedIncomingMessage> => {
-  let done = plugins.smartpromise.defer<any>();
+  const done = plugins.smartpromise.defer<any>();
+
+  // merge options
+  const defaultOptions: interfaces.ISmartRequestOptions = {
+    // agent: agent,
+    autoJsonParse: true,
+    keepAlive: true
+  };
+
+  optionsArg = {
+    ...defaultOptions,
+    ...optionsArg
+  };
 
   // parse url
-  let parsedUrl: plugins.url.Url;
+  const parsedUrl = plugins.url.parse(domainArg);
-  parsedUrl = plugins.url.parse(domainArg);
   optionsArg.hostname = parsedUrl.hostname;
   if (parsedUrl.port) {
-    optionsArg.port = parseInt(parsedUrl.port);
+    optionsArg.port = parseInt(parsedUrl.port, 10);
   }
   optionsArg.path = parsedUrl.path;
 
   // determine if unixsock
-  if(testForUnixSock(domainArg)) {
+  if (testForUnixSock(domainArg)) {
-    const detailedUnixPath = parseSocketPathAndRoute(optionsArg.path)
+    const detailedUnixPath = parseSocketPathAndRoute(optionsArg.path);
     optionsArg.socketPath = detailedUnixPath.socketPath;
     optionsArg.path = detailedUnixPath.path;
   }
 
+  // TODO: support tcp sockets
+
   // lets determine the request module to use
   const requestModule = (() => {
-    if (parsedUrl.protocol === "https:") {
+    switch (true) {
-      return plugins.https;
+      case parsedUrl.protocol === 'https:' && optionsArg.keepAlive:
-    } else if (parsedUrl.protocol === "http:") {
+        optionsArg.agent = httpsAgent;
-      return plugins.http;
+        return plugins.https;
-    } else {
+      case parsedUrl.protocol === 'https:' && !optionsArg.keepAlive:
-      throw new Error(`unsupported protocol: ${parsedUrl.protocol}`);
+        optionsArg.agent = httpsAgentKeepAliveFalse;
+        return plugins.https;
+      case parsedUrl.protocol === 'http:' && optionsArg.keepAlive:
+        optionsArg.agent = httpAgent;
+        return plugins.http;
+      case parsedUrl.protocol === 'http:' && !optionsArg.keepAlive:
+        optionsArg.agent = httpAgentKeepAliveFalse;
+        return plugins.http;
     }
   })() as typeof plugins.https;
 
-
   // lets perform the actual request
-  let request = requestModule.request(optionsArg);
+  const requestToFire = requestModule.request(optionsArg, async response => {
+    if (responseStreamArg) {
+      done.resolve(response);
+    } else {
+      const builtResponse = await buildUtf8Response(response, optionsArg.autoJsonParse);
+      done.resolve(builtResponse);
+    }
+  });
 
   // lets write the requestBody
   if (optionsArg.requestBody) {
-    if (!(optionsArg.requestBody instanceof plugins.formData)) {
+    if (optionsArg.requestBody instanceof plugins.formData) {
-      if(typeof optionsArg.requestBody !== "string") {
+      optionsArg.requestBody.pipe(requestToFire).on('finish', event => {
+        requestToFire.end();
+      });
+    } else {
+      if (typeof optionsArg.requestBody !== 'string') {
         optionsArg.requestBody = JSON.stringify(optionsArg.requestBody);
       }
-      request.write(optionsArg.requestBody);
+      requestToFire.write(optionsArg.requestBody);
-      request.end();
+      requestToFire.end();
-    } else if (optionsArg.requestBody instanceof plugins.formData) {
-      optionsArg.requestBody.pipe(request).on('finish', event => {
-        request.end();
-      });
     }
+  } else if (requestDataFunc) {
+    requestDataFunc(requestToFire);
   } else {
-    request.end();
+    requestToFire.end();
   }
 
   // lets handle an error
-  request.on("error", e => {
+  requestToFire.on('error', e => {
     console.error(e);
   });
 
-  // lets handle the response
-  request.on("response", async response => {
-    if (streamArg) {
-      done.resolve(response);
-    } else {
-      const builtResponse = await buildUtf8Response(response)
-      done.resolve(builtResponse);
-    }
-  });
-
   const result = await done.promise;
   return result;
 };

tslint.json

@@ -1,3 +1,17 @@
 {
-    "extends": "tslint-config-standard"
+  "extends": ["tslint:latest", "tslint-config-prettier"],
+  "rules": {
+    "semicolon": [true, "always"],
+    "no-console": false,
+    "ordered-imports": false,
+    "object-literal-sort-keys": false,
+    "member-ordering": {
+      "options":{
+        "order": [
+          "static-method"
+        ]
+      }
+    }
+  },
+  "defaultSeverity": "warning"
 }