1.0.12

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/abuse.ch",
-  "version": "1.0.2",
+  "version": "1.0.12",
   "private": false,
   "description": "an unofficial client to retrieve abuse.ch data",
   "main": "dist_ts/index.js",
@@ -36,5 +36,13 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
-  ]
+  ],
+  "dependencies": {
+    "@push.rocks/smartfile": "^10.0.28",
+    "@push.rocks/smartpath": "^5.0.5",
+    "csv-parser": "^3.0.0",
+    "https-proxy-agent": "^7.0.1",
+    "node-fetch": "^3.3.2",
+    "unzipper": "^0.10.14"
+  }
 }

test/test.ts

@@ -1,8 +1,23 @@
 import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
 import * as abuseCh from '../ts/index.js';
 
-tap.test('first test', async () => {
+tap.test('should deal with UrlHouse data', async () => {
-  console.log(abuseCh);
+  const urlHouse = new abuseCh.UrlHouse();
+  const data = await urlHouse.getData();
+  console.log(data.length);
+});
+
+tap.test('should deal with UrlHouse data', async () => {
+  const threatFox = new abuseCh.ThreatFox();
+  const data = await threatFox.getData();
+  console.log(data.length);
+});
+
+tap.test('should deal with FeodoTracker data', async () => {
+  const feodoTracker = new abuseCh.FeodoTracker();
+  const data = await feodoTracker.getData();
+  console.log(data.length);
+  console.log(data[1]);
 });
 
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/abuse.ch',
-  version: '1.0.2',
+  version: '1.0.12',
   description: 'an unofficial client to retrieve abuse.ch data'
 }

ts/abuse.ch.classes.feodotracker.ts

@@ -0,0 +1,52 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import * as helpers from './helpers.js';
+
+export interface IFeodoTrackerData {
+  ip_address: string;
+  port: number;
+  status: string;
+  hostname: string | null;
+  as_number: number;
+  as_name: string;
+  country: string;
+  first_seen: string;
+  last_online: string;
+  malware: string;
+}
+
+export class FeodoTracker {
+  private static readonly FEODO_TRACKER_API_URL: string = 'https://feodotracker.abuse.ch/downloads/ipblocklist.json';
+
+  public async getData(): Promise<IFeodoTrackerData[]> {
+    const response = await plugins.nodeFetch(FeodoTracker.FEODO_TRACKER_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data: IFeodoTrackerData[] = await response.json() as IFeodoTrackerData[];
+
+    // Ensure the data is an array and has the expected structure
+    if (!Array.isArray(data) || !data.every(item => 
+      typeof item.ip_address === 'string' &&
+      typeof item.port === 'number' &&
+      (typeof item.hostname === 'string' || item.hostname === null) &&
+      typeof item.as_number === 'number' &&
+      typeof item.as_name === 'string' &&
+      typeof item.country === 'string' &&
+      typeof item.first_seen === 'string' &&
+      typeof item.last_online === 'string' &&
+      typeof item.malware === 'string'
+    )) {
+      throw new Error(`Invalid data structure!`);
+    }
+
+    return data;
+  }
+}
+

ts/abuse.ch.classes.threatfox.ts

@@ -0,0 +1,81 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import * as helpers from './helpers.js';
+
+export interface IThreatFoxData {
+  ID: string;
+  Dateadded: string;
+  URL: string;
+  URLStatus: string;
+  Threat: string;
+  AssociatedTags: string;
+  ThreatFoxLink: string;
+  Reporter: string;
+}
+
+export class ThreatFox {
+  private static readonly THREATFOX_API_URL: string = 'https://threatfox.abuse.ch/export/csv/full'; // Replace with actual API endpoint
+
+  public async getData(): Promise<IThreatFoxData[]> {
+    plugins.smartfile.fs.ensureDirSync(paths.threatFoxTmp);
+    const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
+    const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');
+
+    const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    await new Promise((resolve, reject) => {
+      const fileStream = plugins.fs.createWriteStream(zipPath);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
+    });
+
+    await new Promise((resolve, reject) => {
+      plugins.stream.pipeline(
+        plugins.fs.createReadStream(zipPath),
+        plugins.unzipper.Extract({ path: paths.threatFoxTmp }),
+        (err) => {
+          if (err) reject(err);
+          else resolve(null);
+        }
+      );
+    });
+
+    let data: IThreatFoxData[] = [];
+    await new Promise((resolve, reject) => {
+      plugins.stream.pipeline(
+        plugins.fs.createReadStream(csvPath),
+        plugins.csv({
+          headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'ThreatFoxLink', 'Reporter'],
+          mapValues: ({ header, value }) => value.trim()
+        }),
+        (err) => {
+          if (err) reject(err);
+        }
+      )
+        .on('data', (row) => {
+          data.push(row);
+        })
+        .on('end', resolve)
+        .on('error', reject);
+    });
+    data = data.map((item) => {
+      return {
+        ...item,
+        URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
+      }
+    });
+    plugins.smartfile.fs.removeSync(paths.threatFoxTmp);
+    return data;
+  }
+}

ts/abuse.ch.classes.urlhouse.ts

@@ -0,0 +1,81 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import * as helpers from './helpers.js';
+
+export interface IUrlHouseData {
+  ID: string;
+  Dateadded: string;
+  URL: string;
+  URLStatus: string;
+  Threat: string;
+  AssociatedTags: string;
+  UrlHausLink: string;
+  Reporter: string;
+}
+
+export class UrlHouse {
+  private static readonly URLHOUSE_API_URL: string = 'https://urlhaus.abuse.ch/downloads/csv/';
+
+  public async getData(): Promise<IUrlHouseData[]> {
+    plugins.smartfile.fs.ensureDirSync(paths.urlHouseTmp);
+    const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
+    const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
+
+    const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
+      ...(helpers.findProxy() ? { 
+        agent: helpers.getAgent(),
+      } : {})
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    await new Promise((resolve, reject) => {
+      const fileStream = plugins.fs.createWriteStream(zipPath);
+      // @ts-ignore
+      const readable = plugins.stream.Readable.from(response.body);
+      plugins.stream.pipeline(readable, fileStream, (err) => {
+        if (err) reject(err);
+        else resolve(null);
+      });
+    });
+
+    await new Promise((resolve, reject) => {
+      plugins.stream.pipeline(
+        plugins.fs.createReadStream(zipPath),
+        plugins.unzipper.Extract({ path: paths.urlHouseTmp }),
+        (err) => {
+          if (err) reject(err);
+          else resolve(null);
+        }
+      );
+    });
+
+    let data: IUrlHouseData[] = [];
+    await new Promise((resolve, reject) => {
+      plugins.stream.pipeline(
+        plugins.fs.createReadStream(csvPath),
+        plugins.csv({
+          headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'UrlHausLink', 'Reporter'],
+          mapValues: ({ header, value }) => value.trim()
+        }),
+        (err) => {
+          if (err) reject(err);
+        }
+      )
+        .on('data', (row) => {
+          data.push(row);
+        })
+        .on('end', resolve)
+        .on('error', reject);
+    });
+    data = data.map((item) => {
+      return {
+        ...item,
+        URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
+      }
+    });
+    plugins.smartfile.fs.removeSync(paths.urlHouseTmp);
+    return data;
+  }
+}

ts/helpers.ts

@@ -0,0 +1,8 @@
+import * as plugins from './plugins.js';
+
+export const findProxy = () => {
+  return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
+}
+export const getAgent = () => {
+  return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
+}

ts/index.ts

@@ -1,3 +1,3 @@
-import * as plugins from './abuse.ch.plugins.js';
+export * from './abuse.ch.classes.feodotracker.js';
-
+export * from './abuse.ch.classes.threatfox.js';
-export let demoExport = 'Hi there! :) This is an exported string';
+export * from './abuse.ch.classes.urlhouse.js';

ts/paths.ts

@@ -0,0 +1,8 @@
+import * as plugins from './plugins.js';
+
+export const packageDir = plugins.path.join(plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../');
+export const nogitDir = plugins.path.join(packageDir, '.nogit');
+
+export const urlHouseTmp = plugins.path.join(nogitDir, 'tmp.urlhaus');
+export const threatFoxTmp = plugins.path.join(nogitDir, 'tmp.threatfox');
+export const feodoTrackerTmp = plugins.path.join(nogitDir, 'tmp.feodotracker');

ts/plugins.ts

@@ -0,0 +1,34 @@
+// node native
+import * as stream from 'stream';
+import * as util from 'util';
+import unzipper from 'unzipper';
+import csv from 'csv-parser';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export {
+  stream,
+  util,
+  unzipper,
+  csv,
+  fs,
+  path,
+}
+
+// @push.rocks scope
+import * as smartfile from '@push.rocks/smartfile';
+import * as smartpath from '@push.rocks/smartpath';
+
+export {
+  smartfile,
+  smartpath,
+}
+
+// third party
+import nodeFetch from 'node-fetch';
+import * as httpsProxy from 'https-proxy-agent';
+
+export {
+  nodeFetch,
+  httpsProxy,
+}