fix(core): update

2023-07-28 14:13:16 +02:00 · 2023-07-28 14:13:16 +02:00 · 792224c96b
commit 792224c96b
parent 0517f95d25
7 changed files with 116 additions and 36 deletions
--- a/package.json
+++ b/package.json
@ -40,8 +40,9 @@
  "dependencies": {
    "@push.rocks/smartfile": "^10.0.28",
    "@push.rocks/smartpath": "^5.0.5",
-    "axios": "^1.4.0",
    "csv-parser": "^3.0.0",
+    "https-proxy-agent": "^7.0.1",
+    "node-fetch": "^3.3.2",
    "unzipper": "^0.10.14"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -11,12 +11,15 @@ dependencies:
  '@push.rocks/smartpath':
    specifier: ^5.0.5
    version: 5.0.11
-  axios:
-    specifier: ^1.4.0
-    version: 1.4.0
  csv-parser:
    specifier: ^3.0.0
    version: 3.0.0
+  https-proxy-agent:
+    specifier: ^7.0.1
+    version: 7.0.1
+  node-fetch:
+    specifier: ^3.3.2
+    version: 3.3.2
  unzipper:
    specifier: ^0.10.14
    version: 0.10.14
@ -1910,6 +1913,15 @@ packages:
      - supports-color
    dev: true

+  /agent-base@7.1.0:
+    resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==}
+    engines: {node: '>= 14'}
+    dependencies:
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  /agentkeepalive@4.3.0:
    resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==}
    engines: {node: '>= 8.0.0'}
@ -2012,16 +2024,6 @@ packages:
    engines: {node: '>=4'}
    dev: true

-  /axios@1.4.0:
-    resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==}
-    dependencies:
-      follow-redirects: 1.15.2
-      form-data: 4.0.0
-      proxy-from-env: 1.1.0
-    transitivePeerDependencies:
-      - debug
-    dev: false
-
  /balanced-match@1.0.2:
    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}

@ -2412,6 +2414,11 @@ packages:
      minimist: 1.2.8
    dev: false

+  /data-uri-to-buffer@4.0.1:
+    resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
+    engines: {node: '>= 12'}
+    dev: false
+
  /dayjs@1.11.9:
    resolution: {integrity: sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==}

@ -2818,6 +2825,14 @@ packages:
      xml-js: 1.6.11
    dev: true

+  /fetch-blob@3.2.0:
+    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.2.1
+    dev: false
+
  /figures@5.0.0:
    resolution: {integrity: sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==}
    engines: {node: '>=14'}
@ -2856,16 +2871,6 @@ packages:
      path-exists: 4.0.0
    dev: true

-  /follow-redirects@1.15.2:
-    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
-    engines: {node: '>=4.0'}
-    peerDependencies:
-      debug: '*'
-    peerDependenciesMeta:
-      debug:
-        optional: true
-    dev: false
-
  /foreground-child@2.0.0:
    resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==}
    engines: {node: '>=8.0.0'}
@ -2894,6 +2899,13 @@ packages:
      combined-stream: 1.0.8
      mime-types: 2.1.35

+  /formdata-polyfill@4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.2.0
+    dev: false
+
  /forwarded@0.2.0:
    resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
    engines: {node: '>= 0.6'}
@ -3147,6 +3159,16 @@ packages:
      - supports-color
    dev: true

+  /https-proxy-agent@7.0.1:
+    resolution: {integrity: sha512-Eun8zV0kcYS1g19r78osiQLEFIRspRUDd9tIfBCTBPBeMieF/EsJNL8VI3xOIdYRDEkjQnqOYPsZ2DsWsVsFwQ==}
+    engines: {node: '>= 14'}
+    dependencies:
+      agent-base: 7.1.0
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  /humanize-ms@1.2.1:
    resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=}
    dependencies:
@ -3795,6 +3817,11 @@ packages:
      lower-case: 1.1.4
    dev: true

+  /node-domexception@1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
  /node-fetch@2.6.7:
    resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==}
    engines: {node: 4.x || >=6.0.0}
@ -3807,6 +3834,15 @@ packages:
      whatwg-url: 5.0.0
    dev: true

+  /node-fetch@3.3.2:
+    resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.1
+      fetch-blob: 3.2.0
+      formdata-polyfill: 4.0.10
+    dev: false
+
  /normalize-newline@4.1.0:
    resolution: {integrity: sha512-ff4jKqMI8Xl50/4Mms/9jPobzAV/UK+kXG2XJ/7AqOmxIx8mqfqTIHYxuAnEgJ2AQeBbLnlbmZ5+38Y9A0w/YA==}
    engines: {node: '>=12'}
@ -4045,6 +4081,7 @@ packages:

  /proxy-from-env@1.1.0:
    resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+    dev: true

  /public-ip@6.0.1:
    resolution: {integrity: sha512-1/Mxa1MKrAQ4jF5IalECSBtB0W1FAtnG+9c5X16jjvV/Gx9fiRy7xXIrHlBGYjnTlai0zdZkM3LrpmASavmAEg==}
@ -4819,6 +4856,11 @@ packages:
    resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
    dev: true

+  /web-streams-polyfill@3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: false
+
  /webidl-conversions@3.0.1:
    resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=}
    dev: true
--- a/ts/00_commitinfo_data.ts
+++ b/ts/00_commitinfo_data.ts
@ -3,6 +3,6 @@
 */
 export const commitinfo = {
  name: '@apiclient.xyz/abuse.ch',
-  version: '1.0.6',
+  version: '1.0.7',
  description: 'an unofficial client to retrieve abuse.ch data'
 }
--- a/ts/abuse.ch.classes.threatfox.ts
+++ b/ts/abuse.ch.classes.threatfox.ts
@ -1,6 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
-import axios from 'axios';
+import * as helpers from './helpers.js';

 export interface IThreatFoxData {
  ID: string;
@ -21,13 +21,23 @@ export class ThreatFox {
    const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
    const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');

-    const response = await axios.get(ThreatFox.THREATFOX_API_URL, { responseType: 'stream' });
-    
+    const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
    await new Promise((resolve, reject) => {
      const fileStream = plugins.fs.createWriteStream(zipPath);
-      response.data.pipe(fileStream);
-      fileStream.on('finish', resolve);
-      fileStream.on('error', reject);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
    });

    await new Promise((resolve, reject) => {
--- a/ts/abuse.ch.classes.urlhouse.ts
+++ b/ts/abuse.ch.classes.urlhouse.ts
@ -1,6 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
-import axios from 'axios';
+import * as helpers from './helpers.js';

 export interface IUrlHouseData {
  ID: string;
@ -21,13 +21,23 @@ export class UrlHouse {
    const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
    const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');

-    const response = await axios.get(UrlHouse.URLHOUSE_API_URL, { responseType: 'stream' });
+    const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }

    await new Promise((resolve, reject) => {
      const fileStream = plugins.fs.createWriteStream(zipPath);
-      response.data.pipe(fileStream);
-      fileStream.on('finish', resolve);
-      fileStream.on('error', reject);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
    });

    await new Promise((resolve, reject) => {
--- a/ts/helpers.ts
+++ b/ts/helpers.ts
@ -0,0 +1,8 @@
+import * as plugins from './plugins.js';
+
+export const findProxy = () => {
+  return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
+}
+export const getAgent = () => {
+  return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
+}
--- a/ts/plugins.ts
+++ b/ts/plugins.ts
@ -22,4 +22,13 @@ import * as smartpath from '@push.rocks/smartpath';
 export {
  smartfile,
  smartpath,
+}
+
+// third party
+import nodeFetch from 'node-fetch';
+import * as httpsProxy from 'https-proxy-agent';
+
+export {
+  nodeFetch,
+  httpsProxy,
 }