1.0.9

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/abuse.ch",
-  "version": "1.0.4",
+  "version": "1.0.9",
   "private": false,
   "description": "an unofficial client to retrieve abuse.ch data",
   "main": "dist_ts/index.js",
@@ -41,6 +41,8 @@
     "@push.rocks/smartfile": "^10.0.28",
     "@push.rocks/smartpath": "^5.0.5",
     "csv-parser": "^3.0.0",
+    "https-proxy-agent": "^7.0.1",
+    "node-fetch": "^3.3.2",
     "unzipper": "^0.10.14"
   }
 }

pnpm-lock.yaml

@@ -14,6 +14,12 @@ dependencies:
   csv-parser:
     specifier: ^3.0.0
     version: 3.0.0
+  https-proxy-agent:
+    specifier: ^7.0.1
+    version: 7.0.1
+  node-fetch:
+    specifier: ^3.3.2
+    version: 3.3.2
   unzipper:
     specifier: ^0.10.14
     version: 0.10.14
@@ -1907,6 +1913,15 @@ packages:
       - supports-color
     dev: true
 
+  /agent-base@7.1.0:
+    resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==}
+    engines: {node: '>= 14'}
+    dependencies:
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /agentkeepalive@4.3.0:
     resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==}
     engines: {node: '>= 8.0.0'}
@@ -2399,6 +2414,11 @@ packages:
       minimist: 1.2.8
     dev: false
 
+  /data-uri-to-buffer@4.0.1:
+    resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
+    engines: {node: '>= 12'}
+    dev: false
+
   /dayjs@1.11.9:
     resolution: {integrity: sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==}
 
@@ -2805,6 +2825,14 @@ packages:
       xml-js: 1.6.11
     dev: true
 
+  /fetch-blob@3.2.0:
+    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.2.1
+    dev: false
+
   /figures@5.0.0:
     resolution: {integrity: sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==}
     engines: {node: '>=14'}
@@ -2871,6 +2899,13 @@ packages:
       combined-stream: 1.0.8
       mime-types: 2.1.35
 
+  /formdata-polyfill@4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.2.0
+    dev: false
+
   /forwarded@0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -3124,6 +3159,16 @@ packages:
       - supports-color
     dev: true
 
+  /https-proxy-agent@7.0.1:
+    resolution: {integrity: sha512-Eun8zV0kcYS1g19r78osiQLEFIRspRUDd9tIfBCTBPBeMieF/EsJNL8VI3xOIdYRDEkjQnqOYPsZ2DsWsVsFwQ==}
+    engines: {node: '>= 14'}
+    dependencies:
+      agent-base: 7.1.0
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /humanize-ms@1.2.1:
     resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=}
     dependencies:
@@ -3772,6 +3817,11 @@ packages:
       lower-case: 1.1.4
     dev: true
 
+  /node-domexception@1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
   /node-fetch@2.6.7:
     resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==}
     engines: {node: 4.x || >=6.0.0}
@@ -3784,6 +3834,15 @@ packages:
       whatwg-url: 5.0.0
     dev: true
 
+  /node-fetch@3.3.2:
+    resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.1
+      fetch-blob: 3.2.0
+      formdata-polyfill: 4.0.10
+    dev: false
+
   /normalize-newline@4.1.0:
     resolution: {integrity: sha512-ff4jKqMI8Xl50/4Mms/9jPobzAV/UK+kXG2XJ/7AqOmxIx8mqfqTIHYxuAnEgJ2AQeBbLnlbmZ5+38Y9A0w/YA==}
     engines: {node: '>=12'}
@@ -4797,6 +4856,11 @@ packages:
     resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
     dev: true
 
+  /web-streams-polyfill@3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: false
+
   /webidl-conversions@3.0.1:
     resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=}
     dev: true

test/test.ts

@@ -13,4 +13,11 @@ tap.test('should deal with UrlHouse data', async () => {
   console.log(data.length);
 });
 
+tap.test('should deal with FeodoTracker data', async () => {
+  const feodoTracker = new abuseCh.FeodoTracker();
+  const data = await feodoTracker.getData();
+  console.log(data.length);
+  console.log(data[0]);
+});
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/abuse.ch',
-  version: '1.0.4',
+  version: '1.0.9',
   description: 'an unofficial client to retrieve abuse.ch data'
 }

ts/abuse.ch.classes.feodotracker.ts

@@ -0,0 +1,63 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import * as helpers from './helpers.js';
+
+export interface IFeodoTrackerData {
+  ID: string;
+  Dateadded: string;
+  DestinationIP: string;
+  DestinationPort: string;
+  Malware: string;
+  Reporter: string;
+}
+
+
+export class FeodoTracker {
+  private static readonly FEODO_TRACKER_API_URL: string = 'https://feodotracker.abuse.ch/downloads/ipblocklist_recommended.txt';
+
+  public async getData(): Promise<IFeodoTrackerData[]> {
+    plugins.smartfile.fs.ensureDirSync(paths.feodoTrackerTmp);
+    const txtPath = plugins.path.join(paths.feodoTrackerTmp, 'feodo.txt');
+
+    const response = await plugins.nodeFetch(FeodoTracker.FEODO_TRACKER_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    await new Promise((resolve, reject) => {
+      const fileStream = plugins.fs.createWriteStream(txtPath);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
+    });
+
+    let data: IFeodoTrackerData[] = [];
+    await new Promise((resolve, reject) => {
+      plugins.stream.pipeline(
+        plugins.fs.createReadStream(txtPath),
+        plugins.csv({
+          headers: ['ID', 'Dateadded', 'DestinationIP', 'DestinationPort', 'Malware', 'Reporter'],
+          mapValues: ({ header, value }) => value.trim()
+        }),
+        (err) => {
+          if (err) reject(err);
+        }
+      )
+        .on('data', (row) => {
+          data.push(row);
+        })
+        .on('end', resolve)
+        .on('error', reject);
+    });
+
+    plugins.smartfile.fs.removeSync(paths.feodoTrackerTmp);
+    return data;
+  }
+}

ts/abuse.ch.classes.threatfox.ts

@@ -1,5 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
+import * as helpers from './helpers.js';
 
 export interface IThreatFoxData {
   ID: string;
@@ -20,7 +21,11 @@ export class ThreatFox {
     const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
     const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');
 
-    const response = await fetch(ThreatFox.THREATFOX_API_URL);
+    const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
     if (!response.ok) {
       throw new Error(`HTTP error! status: ${response.status}`);
     }

ts/abuse.ch.classes.urlhouse.ts

@@ -1,5 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
+import * as helpers from './helpers.js';
 
 export interface IUrlHouseData {
   ID: string;
@@ -20,7 +21,11 @@ export class UrlHouse {
     const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
     const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
 
-    const response = await fetch(UrlHouse.URLHOUSE_API_URL);
+    const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
     if (!response.ok) {
       throw new Error(`HTTP error! status: ${response.status}`);
     }

ts/helpers.ts

@@ -0,0 +1,8 @@
+import * as plugins from './plugins.js';
+
+export const findProxy = () => {
+  return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
+}
+export const getAgent = () => {
+  return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
+}

ts/index.ts

@@ -1,2 +1,3 @@
-export * from './abuse.ch.classes.urlhouse.js';
+export * from './abuse.ch.classes.feodotracker.js';
 export * from './abuse.ch.classes.threatfox.js';
+export * from './abuse.ch.classes.urlhouse.js';

ts/paths.ts

@@ -5,3 +5,4 @@ export const nogitDir = plugins.path.join(packageDir, '.nogit');
 
 export const urlHouseTmp = plugins.path.join(nogitDir, 'tmp.urlhaus');
 export const threatFoxTmp = plugins.path.join(nogitDir, 'tmp.threatfox');
+export const feodoTrackerTmp = plugins.path.join(nogitDir, 'tmp.feodotracker');

ts/plugins.ts

@@ -22,4 +22,13 @@ import * as smartpath from '@push.rocks/smartpath';
 export {
   smartfile,
   smartpath,
+}
+
+// third party
+import nodeFetch from 'node-fetch';
+import * as httpsProxy from 'https-proxy-agent';
+
+export {
+  nodeFetch,
+  httpsProxy,
 }