1.0.12

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/abuse.ch",
-  "version": "1.0.6",
+  "version": "1.0.12",
   "private": false,
   "description": "an unofficial client to retrieve abuse.ch data",
   "main": "dist_ts/index.js",
@@ -40,8 +40,9 @@
   "dependencies": {
     "@push.rocks/smartfile": "^10.0.28",
     "@push.rocks/smartpath": "^5.0.5",
-    "axios": "^1.4.0",
     "csv-parser": "^3.0.0",
+    "https-proxy-agent": "^7.0.1",
+    "node-fetch": "^3.3.2",
     "unzipper": "^0.10.14"
   }
 }

pnpm-lock.yaml

@@ -11,12 +11,15 @@ dependencies:
   '@push.rocks/smartpath':
     specifier: ^5.0.5
     version: 5.0.11
-  axios:
-    specifier: ^1.4.0
-    version: 1.4.0
   csv-parser:
     specifier: ^3.0.0
     version: 3.0.0
+  https-proxy-agent:
+    specifier: ^7.0.1
+    version: 7.0.1
+  node-fetch:
+    specifier: ^3.3.2
+    version: 3.3.2
   unzipper:
     specifier: ^0.10.14
     version: 0.10.14
@@ -1910,6 +1913,15 @@ packages:
       - supports-color
     dev: true
 
+  /agent-base@7.1.0:
+    resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==}
+    engines: {node: '>= 14'}
+    dependencies:
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /agentkeepalive@4.3.0:
     resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==}
     engines: {node: '>= 8.0.0'}
@@ -2012,16 +2024,6 @@ packages:
     engines: {node: '>=4'}
     dev: true
 
-  /axios@1.4.0:
-    resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==}
-    dependencies:
-      follow-redirects: 1.15.2
-      form-data: 4.0.0
-      proxy-from-env: 1.1.0
-    transitivePeerDependencies:
-      - debug
-    dev: false
-
   /balanced-match@1.0.2:
     resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
 
@@ -2412,6 +2414,11 @@ packages:
       minimist: 1.2.8
     dev: false
 
+  /data-uri-to-buffer@4.0.1:
+    resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
+    engines: {node: '>= 12'}
+    dev: false
+
   /dayjs@1.11.9:
     resolution: {integrity: sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==}
 
@@ -2818,6 +2825,14 @@ packages:
       xml-js: 1.6.11
     dev: true
 
+  /fetch-blob@3.2.0:
+    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.2.1
+    dev: false
+
   /figures@5.0.0:
     resolution: {integrity: sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==}
     engines: {node: '>=14'}
@@ -2856,16 +2871,6 @@ packages:
       path-exists: 4.0.0
     dev: true
 
-  /follow-redirects@1.15.2:
-    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
-    engines: {node: '>=4.0'}
-    peerDependencies:
-      debug: '*'
-    peerDependenciesMeta:
-      debug:
-        optional: true
-    dev: false
-
   /foreground-child@2.0.0:
     resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==}
     engines: {node: '>=8.0.0'}
@@ -2894,6 +2899,13 @@ packages:
       combined-stream: 1.0.8
       mime-types: 2.1.35
 
+  /formdata-polyfill@4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.2.0
+    dev: false
+
   /forwarded@0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -3147,6 +3159,16 @@ packages:
       - supports-color
     dev: true
 
+  /https-proxy-agent@7.0.1:
+    resolution: {integrity: sha512-Eun8zV0kcYS1g19r78osiQLEFIRspRUDd9tIfBCTBPBeMieF/EsJNL8VI3xOIdYRDEkjQnqOYPsZ2DsWsVsFwQ==}
+    engines: {node: '>= 14'}
+    dependencies:
+      agent-base: 7.1.0
+      debug: 4.3.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /humanize-ms@1.2.1:
     resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=}
     dependencies:
@@ -3795,6 +3817,11 @@ packages:
       lower-case: 1.1.4
     dev: true
 
+  /node-domexception@1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
   /node-fetch@2.6.7:
     resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==}
     engines: {node: 4.x || >=6.0.0}
@@ -3807,6 +3834,15 @@ packages:
       whatwg-url: 5.0.0
     dev: true
 
+  /node-fetch@3.3.2:
+    resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.1
+      fetch-blob: 3.2.0
+      formdata-polyfill: 4.0.10
+    dev: false
+
   /normalize-newline@4.1.0:
     resolution: {integrity: sha512-ff4jKqMI8Xl50/4Mms/9jPobzAV/UK+kXG2XJ/7AqOmxIx8mqfqTIHYxuAnEgJ2AQeBbLnlbmZ5+38Y9A0w/YA==}
     engines: {node: '>=12'}
@@ -4045,6 +4081,7 @@ packages:
 
   /proxy-from-env@1.1.0:
     resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+    dev: true
 
   /public-ip@6.0.1:
     resolution: {integrity: sha512-1/Mxa1MKrAQ4jF5IalECSBtB0W1FAtnG+9c5X16jjvV/Gx9fiRy7xXIrHlBGYjnTlai0zdZkM3LrpmASavmAEg==}
@@ -4819,6 +4856,11 @@ packages:
     resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
     dev: true
 
+  /web-streams-polyfill@3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: false
+
   /webidl-conversions@3.0.1:
     resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=}
     dev: true

test/test.ts

@@ -13,4 +13,11 @@ tap.test('should deal with UrlHouse data', async () => {
   console.log(data.length);
 });
 
+tap.test('should deal with FeodoTracker data', async () => {
+  const feodoTracker = new abuseCh.FeodoTracker();
+  const data = await feodoTracker.getData();
+  console.log(data.length);
+  console.log(data[1]);
+});
+
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/abuse.ch',
-  version: '1.0.6',
+  version: '1.0.12',
   description: 'an unofficial client to retrieve abuse.ch data'
 }

ts/abuse.ch.classes.feodotracker.ts

@@ -0,0 +1,52 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import * as helpers from './helpers.js';
+
+export interface IFeodoTrackerData {
+  ip_address: string;
+  port: number;
+  status: string;
+  hostname: string | null;
+  as_number: number;
+  as_name: string;
+  country: string;
+  first_seen: string;
+  last_online: string;
+  malware: string;
+}
+
+export class FeodoTracker {
+  private static readonly FEODO_TRACKER_API_URL: string = 'https://feodotracker.abuse.ch/downloads/ipblocklist.json';
+
+  public async getData(): Promise<IFeodoTrackerData[]> {
+    const response = await plugins.nodeFetch(FeodoTracker.FEODO_TRACKER_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data: IFeodoTrackerData[] = await response.json() as IFeodoTrackerData[];
+
+    // Ensure the data is an array and has the expected structure
+    if (!Array.isArray(data) || !data.every(item => 
+      typeof item.ip_address === 'string' &&
+      typeof item.port === 'number' &&
+      (typeof item.hostname === 'string' || item.hostname === null) &&
+      typeof item.as_number === 'number' &&
+      typeof item.as_name === 'string' &&
+      typeof item.country === 'string' &&
+      typeof item.first_seen === 'string' &&
+      typeof item.last_online === 'string' &&
+      typeof item.malware === 'string'
+    )) {
+      throw new Error(`Invalid data structure!`);
+    }
+
+    return data;
+  }
+}
+

ts/abuse.ch.classes.threatfox.ts

@@ -1,6 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
-import axios from 'axios';
+import * as helpers from './helpers.js';
 
 export interface IThreatFoxData {
   ID: string;
@@ -21,13 +21,23 @@ export class ThreatFox {
     const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
     const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');
 
-    const response = await axios.get(ThreatFox.THREATFOX_API_URL, { responseType: 'stream' });
-    
+    const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
     await new Promise((resolve, reject) => {
       const fileStream = plugins.fs.createWriteStream(zipPath);
-      response.data.pipe(fileStream);
-      fileStream.on('finish', resolve);
-      fileStream.on('error', reject);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
     });
 
     await new Promise((resolve, reject) => {

ts/abuse.ch.classes.urlhouse.ts

@@ -1,6 +1,6 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
-import axios from 'axios';
+import * as helpers from './helpers.js';
 
 export interface IUrlHouseData {
   ID: string;
@@ -21,13 +21,23 @@ export class UrlHouse {
     const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
     const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
 
-    const response = await axios.get(UrlHouse.URLHOUSE_API_URL, { responseType: 'stream' });
+    const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
 
     await new Promise((resolve, reject) => {
       const fileStream = plugins.fs.createWriteStream(zipPath);
-      response.data.pipe(fileStream);
-      fileStream.on('finish', resolve);
-      fileStream.on('error', reject);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
     });
 
     await new Promise((resolve, reject) => {

ts/helpers.ts

@@ -0,0 +1,8 @@
+import * as plugins from './plugins.js';
+
+export const findProxy = () => {
+  return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
+}
+export const getAgent = () => {
+  return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
+}

ts/index.ts

@@ -1,2 +1,3 @@
-export * from './abuse.ch.classes.urlhouse.js';
+export * from './abuse.ch.classes.feodotracker.js';
 export * from './abuse.ch.classes.threatfox.js';
+export * from './abuse.ch.classes.urlhouse.js';

ts/paths.ts

@@ -5,3 +5,4 @@ export const nogitDir = plugins.path.join(packageDir, '.nogit');
 
 export const urlHouseTmp = plugins.path.join(nogitDir, 'tmp.urlhaus');
 export const threatFoxTmp = plugins.path.join(nogitDir, 'tmp.threatfox');
+export const feodoTrackerTmp = plugins.path.join(nogitDir, 'tmp.feodotracker');

ts/plugins.ts

@@ -22,4 +22,13 @@ import * as smartpath from '@push.rocks/smartpath';
 export {
   smartfile,
   smartpath,
+}
+
+// third party
+import nodeFetch from 'node-fetch';
+import * as httpsProxy from 'https-proxy-agent';
+
+export {
+  nodeFetch,
+  httpsProxy,
 }