add modern version of request construction

ts/legacy/index.ts

@@ -0,0 +1,8 @@
+export { request, safeGet } from './smartrequest.request.js';
+export type { IExtendedIncomingMessage } from './smartrequest.request.js';
+export type { ISmartRequestOptions } from './smartrequest.interfaces.js';
+
+export * from './smartrequest.jsonrest.js';
+export * from './smartrequest.binaryrest.js';
+export * from './smartrequest.formdata.js';
+export * from './smartrequest.stream.js';

ts/legacy/smartrequest.binaryrest.ts

@@ -0,0 +1,33 @@
+// this file implements methods to get and post binary data.
+import * as interfaces from './smartrequest.interfaces.js';
+import { request, type IExtendedIncomingMessage } from './smartrequest.request.js';
+
+import * as plugins from './smartrequest.plugins.js';
+
+export const getBinary = async (
+  domainArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+) => {
+  optionsArg = {
+    ...optionsArg,
+    autoJsonParse: false,
+  };
+  const done = plugins.smartpromise.defer();
+  const response = await request(domainArg, optionsArg, true);
+  const data: Array<Buffer> = [];
+
+  response
+    .on('data', function (chunk: Buffer) {
+      data.push(chunk);
+    })
+    .on('end', function () {
+      //at this point data is an array of Buffers
+      //so Buffer.concat() can make us a new Buffer
+      //of all of them together
+      const buffer = Buffer.concat(data);
+      response.body = buffer;
+      done.resolve();
+    });
+  await done.promise;
+  return response as IExtendedIncomingMessage<Buffer>;
+};

ts/legacy/smartrequest.formdata.ts

@@ -0,0 +1,99 @@
+import * as plugins from './smartrequest.plugins.js';
+import * as interfaces from './smartrequest.interfaces.js';
+import { request } from './smartrequest.request.js';
+
+/**
+ * the interfae for FormFieldData
+ */
+export interface IFormField {
+  name: string;
+  type: 'string' | 'filePath' | 'Buffer';
+  payload: string | Buffer;
+  fileName?: string;
+  contentType?: string;
+}
+
+const appendFormField = async (formDataArg: plugins.formData, formDataField: IFormField) => {
+  switch (formDataField.type) {
+    case 'string':
+      formDataArg.append(formDataField.name, formDataField.payload);
+      break;
+    case 'filePath':
+      if (typeof formDataField.payload !== 'string') {
+        throw new Error(
+          `Payload for key ${
+            formDataField.name
+          } must be of type string. Got ${typeof formDataField.payload} instead.`
+        );
+      }
+      const fileData = plugins.fs.readFileSync(
+        plugins.path.join(process.cwd(), formDataField.payload)
+      );
+      formDataArg.append('file', fileData, {
+        filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf',
+        contentType: 'application/pdf',
+      });
+      break;
+    case 'Buffer':
+      formDataArg.append(formDataField.name, formDataField.payload, {
+        filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf',
+        contentType: formDataField.contentType ? formDataField.contentType : 'application/pdf',
+      });
+      break;
+  }
+};
+
+export const postFormData = async (
+  urlArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {},
+  payloadArg: IFormField[]
+) => {
+  const form = new plugins.formData();
+  for (const formField of payloadArg) {
+    await appendFormField(form, formField);
+  }
+  const requestOptions = {
+    ...optionsArg,
+    method: 'POST',
+    headers: {
+      ...optionsArg.headers,
+      ...form.getHeaders(),
+    },
+    requestBody: form,
+  };
+
+  // lets fire the actual request for sending the formdata
+  const response = await request(urlArg, requestOptions);
+  return response;
+};
+
+export const postFormDataUrlEncoded = async (
+  urlArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {},
+  payloadArg: { key: string; content: string }[]
+) => {
+  let resultString = '';
+
+  for (const keyContentPair of payloadArg) {
+    if (resultString) {
+      resultString += '&';
+    }
+    resultString += `${encodeURIComponent(keyContentPair.key)}=${encodeURIComponent(
+      keyContentPair.content
+    )}`;
+  }
+
+  const requestOptions: interfaces.ISmartRequestOptions = {
+    ...optionsArg,
+    method: 'POST',
+    headers: {
+      ...optionsArg.headers,
+      'content-type': 'application/x-www-form-urlencoded',
+    },
+    requestBody: resultString,
+  };
+
+  // lets fire the actual request for sending the formdata
+  const response = await request(urlArg, requestOptions);
+  return response;
+};

ts/legacy/smartrequest.interfaces.ts

@@ -0,0 +1,10 @@
+import * as plugins from './smartrequest.plugins.js';
+import * as https from 'https';
+
+export interface ISmartRequestOptions extends https.RequestOptions {
+  keepAlive?: boolean;
+  requestBody?: any;
+  autoJsonParse?: boolean;
+  queryParams?: { [key: string]: string };
+  hardDataCuttingTimeout?: number;
+}

ts/legacy/smartrequest.jsonrest.ts

@@ -0,0 +1,63 @@
+// This file implements methods to get and post JSON in a simple manner.
+
+import * as interfaces from './smartrequest.interfaces.js';
+import { request } from './smartrequest.request.js';
+
+/**
+ * gets Json and puts the right headers + handles response aggregation
+ * @param domainArg
+ * @param optionsArg
+ */
+export const getJson = async (
+  domainArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+) => {
+  optionsArg.method = 'GET';
+  optionsArg.headers = {
+    ...optionsArg.headers,
+  };
+  let response = await request(domainArg, optionsArg);
+  return response;
+};
+
+export const postJson = async (
+  domainArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+) => {
+  optionsArg.method = 'POST';
+  if (
+    typeof optionsArg.requestBody === 'object' &&
+    (!optionsArg.headers || !optionsArg.headers['Content-Type'])
+  ) {
+    // make sure headers exist
+    if (!optionsArg.headers) {
+      optionsArg.headers = {};
+    }
+
+    // assign the right Content-Type, leaving all other headers in place
+    optionsArg.headers = {
+      ...optionsArg.headers,
+      'Content-Type': 'application/json',
+    };
+  }
+  let response = await request(domainArg, optionsArg);
+  return response;
+};
+
+export const putJson = async (
+  domainArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+) => {
+  optionsArg.method = 'PUT';
+  let response = await request(domainArg, optionsArg);
+  return response;
+};
+
+export const delJson = async (
+  domainArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+) => {
+  optionsArg.method = 'DELETE';
+  let response = await request(domainArg, optionsArg);
+  return response;
+};

ts/legacy/smartrequest.plugins.ts

@@ -0,0 +1,19 @@
+// node native scope
+import * as fs from 'fs';
+import * as http from 'http';
+import * as https from 'https';
+import * as path from 'path';
+
+export { http, https, fs, path };
+
+// pushrocks scope
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smarturl from '@push.rocks/smarturl';
+
+export { smartpromise, smarturl };
+
+// third party scope
+import agentkeepalive from 'agentkeepalive';
+import formData from 'form-data';
+
+export { agentkeepalive, formData };

ts/legacy/smartrequest.request.ts

@@ -0,0 +1,231 @@
+import * as plugins from './smartrequest.plugins.js';
+import * as interfaces from './smartrequest.interfaces.js';
+
+export interface IExtendedIncomingMessage<T = any> extends plugins.http.IncomingMessage {
+  body: T;
+}
+
+const buildUtf8Response = (
+  incomingMessageArg: plugins.http.IncomingMessage,
+  autoJsonParse = true
+): Promise<IExtendedIncomingMessage> => {
+  const done = plugins.smartpromise.defer<IExtendedIncomingMessage>();
+  // Continuously update stream with data
+  let body = '';
+  incomingMessageArg.on('data', (chunkArg) => {
+    body += chunkArg;
+  });
+
+  incomingMessageArg.on('end', () => {
+    if (autoJsonParse) {
+      try {
+        (incomingMessageArg as IExtendedIncomingMessage).body = JSON.parse(body);
+      } catch (err) {
+        (incomingMessageArg as IExtendedIncomingMessage).body = body;
+      }
+    } else {
+      (incomingMessageArg as IExtendedIncomingMessage).body = body;
+    }
+    done.resolve(incomingMessageArg as IExtendedIncomingMessage);
+  });
+  return done.promise;
+};
+
+/**
+ * determine wether a url is a unix sock
+ * @param urlArg
+ */
+const testForUnixSock = (urlArg: string): boolean => {
+  const unixRegex = /^(http:\/\/|https:\/\/|)unix:/;
+  return unixRegex.test(urlArg);
+};
+
+/**
+ * determine socketPath and path for unixsock
+ */
+const parseSocketPathAndRoute = (stringToParseArg: string) => {
+  const parseRegex = /(.*):(.*)/;
+  const result = parseRegex.exec(stringToParseArg);
+  return {
+    socketPath: result[1],
+    path: result[2],
+  };
+};
+
+/**
+ * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpAgent = new plugins.agentkeepalive({
+  keepAlive: true,
+  maxFreeSockets: 10,
+  maxSockets: 100,
+  maxTotalSockets: 1000,
+  timeout: 60000,
+});
+
+/**
+ * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpAgentKeepAliveFalse = new plugins.agentkeepalive({
+  keepAlive: false,
+  timeout: 60000,
+});
+
+/**
+ * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpsAgent = new plugins.agentkeepalive.HttpsAgent({
+  keepAlive: true,
+  maxFreeSockets: 10,
+  maxSockets: 100,
+  maxTotalSockets: 1000,
+  timeout: 60000,
+});
+
+/**
+ * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
+ */
+const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({
+  keepAlive: false,
+  timeout: 60000,
+});
+
+export let request = async (
+  urlArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {},
+  responseStreamArg: boolean = false,
+  requestDataFunc: (req: plugins.http.ClientRequest) => void = null
+): Promise<IExtendedIncomingMessage> => {
+  const done = plugins.smartpromise.defer<IExtendedIncomingMessage>();
+
+  // merge options
+  const defaultOptions: interfaces.ISmartRequestOptions = {
+    // agent: agent,
+    autoJsonParse: true,
+    keepAlive: true,
+  };
+
+  optionsArg = {
+    ...defaultOptions,
+    ...optionsArg,
+  };
+
+  // parse url
+  const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(urlArg, {
+    searchParams: optionsArg.queryParams || {},
+  });
+  optionsArg.hostname = parsedUrl.hostname;
+  if (parsedUrl.port) {
+    optionsArg.port = parseInt(parsedUrl.port, 10);
+  }
+  optionsArg.path = parsedUrl.path;
+  optionsArg.queryParams = parsedUrl.searchParams;
+
+  // determine if unixsock
+  if (testForUnixSock(urlArg)) {
+    const detailedUnixPath = parseSocketPathAndRoute(optionsArg.path);
+    optionsArg.socketPath = detailedUnixPath.socketPath;
+    optionsArg.path = detailedUnixPath.path;
+  }
+
+  // TODO: support tcp sockets
+
+  // lets determine agent
+  switch (true) {
+    case !!optionsArg.agent:
+      break;
+    case parsedUrl.protocol === 'https:' && optionsArg.keepAlive:
+      optionsArg.agent = httpsAgent;
+      break;
+    case parsedUrl.protocol === 'https:' && !optionsArg.keepAlive:
+      optionsArg.agent = httpsAgentKeepAliveFalse;
+      break;
+    case parsedUrl.protocol === 'http:' && optionsArg.keepAlive:
+      optionsArg.agent = httpAgent;
+      break;
+    case parsedUrl.protocol === 'http:' && !optionsArg.keepAlive:
+      optionsArg.agent = httpAgentKeepAliveFalse;
+      break;
+  }
+
+  // lets determine the request module to use
+  const requestModule = (() => {
+    switch (true) {
+      case parsedUrl.protocol === 'https:':
+        return plugins.https;
+      case parsedUrl.protocol === 'http:':
+        return plugins.http;
+    }
+  })() as typeof plugins.https;
+
+  if (!requestModule) {
+    console.error(`The request to ${urlArg} is missing a viable protocol. Must be http or https`);
+    return;
+  }
+
+  // lets perform the actual request
+  const requestToFire = requestModule.request(optionsArg, async (resArg) => {
+    if (optionsArg.hardDataCuttingTimeout) {
+      setTimeout(() => {
+        resArg.destroy();
+        done.reject(new Error('Request timed out'));
+      }, optionsArg.hardDataCuttingTimeout)
+    }
+
+    if (responseStreamArg) {
+      done.resolve(resArg as IExtendedIncomingMessage);
+    } else {
+      const builtResponse = await buildUtf8Response(resArg, optionsArg.autoJsonParse);
+      done.resolve(builtResponse);
+    }
+  });
+
+  // lets write the requestBody
+  if (optionsArg.requestBody) {
+    if (optionsArg.requestBody instanceof plugins.formData) {
+      optionsArg.requestBody.pipe(requestToFire).on('finish', (event: any) => {
+        requestToFire.end();
+      });
+    } else {
+      if (typeof optionsArg.requestBody !== 'string') {
+        optionsArg.requestBody = JSON.stringify(optionsArg.requestBody);
+      }
+      requestToFire.write(optionsArg.requestBody);
+      requestToFire.end();
+    }
+  } else if (requestDataFunc) {
+    requestDataFunc(requestToFire);
+  } else {
+    requestToFire.end();
+  }
+
+  // lets handle an error
+  requestToFire.on('error', (e) => {
+    console.error(e);
+    requestToFire.destroy();
+  });
+
+  const response = await done.promise;
+  response.on('error', (err) => {
+    console.log(err);
+    response.destroy();
+  });
+  return response;
+};
+
+export const safeGet = async (urlArg: string) => {
+  const agentToUse = urlArg.startsWith('http://') ? new plugins.http.Agent() : new plugins.https.Agent();
+  try {
+    const response = await request(urlArg, {
+      method: 'GET',
+      agent: agentToUse,
+      timeout: 5000,
+      hardDataCuttingTimeout: 5000,
+      autoJsonParse: false,
+    });
+    return response;
+  } catch (err) {
+    console.log(err);
+    return null;
+  }
+};

ts/legacy/smartrequest.stream.ts

@@ -0,0 +1,17 @@
+import * as plugins from './smartrequest.plugins.js';
+import * as interfaces from './smartrequest.interfaces.js';
+import { request } from './smartrequest.request.js';
+
+export const getStream = async (
+  urlArg: string,
+  optionsArg: interfaces.ISmartRequestOptions = {}
+): Promise<plugins.http.IncomingMessage> => {
+  try {
+    // Call the existing request function with responseStreamArg set to true.
+    const responseStream = await request(urlArg, optionsArg, true);
+    return responseStream;
+  } catch (err) {
+    console.error('An error occurred while getting the stream:', err);
+    throw err; // Rethrow the error to be handled by the caller.
+  }
+};