package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apiclient.xyz/abuse.ch",
-  "version": "1.0.12",
+  "version": "1.0.2",
   "private": false,
   "description": "an unofficial client to retrieve abuse.ch data",
   "main": "dist_ts/index.js",
@@ -36,13 +36,5 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
-  ],
+  ]
-  "dependencies": {
-    "@push.rocks/smartfile": "^10.0.28",
-    "@push.rocks/smartpath": "^5.0.5",
-    "csv-parser": "^3.0.0",
-    "https-proxy-agent": "^7.0.1",
-    "node-fetch": "^3.3.2",
-    "unzipper": "^0.10.14"
-  }
 }

test/test.ts

@@ -1,23 +1,8 @@
 import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
 import * as abuseCh from '../ts/index.js';
 
-tap.test('should deal with UrlHouse data', async () => {
+tap.test('first test', async () => {
-  const urlHouse = new abuseCh.UrlHouse();
+  console.log(abuseCh);
-  const data = await urlHouse.getData();
-  console.log(data.length);
-});
-
-tap.test('should deal with UrlHouse data', async () => {
-  const threatFox = new abuseCh.ThreatFox();
-  const data = await threatFox.getData();
-  console.log(data.length);
-});
-
-tap.test('should deal with FeodoTracker data', async () => {
-  const feodoTracker = new abuseCh.FeodoTracker();
-  const data = await feodoTracker.getData();
-  console.log(data.length);
-  console.log(data[1]);
 });
 
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@apiclient.xyz/abuse.ch',
-  version: '1.0.12',
+  version: '1.0.2',
   description: 'an unofficial client to retrieve abuse.ch data'
 }

ts/abuse.ch.classes.feodotracker.ts

@@ -1,52 +0,0 @@
-import * as plugins from './plugins.js';
-import * as paths from './paths.js';
-import * as helpers from './helpers.js';
-
-export interface IFeodoTrackerData {
-  ip_address: string;
-  port: number;
-  status: string;
-  hostname: string | null;
-  as_number: number;
-  as_name: string;
-  country: string;
-  first_seen: string;
-  last_online: string;
-  malware: string;
-}
-
-export class FeodoTracker {
-  private static readonly FEODO_TRACKER_API_URL: string = 'https://feodotracker.abuse.ch/downloads/ipblocklist.json';
-
-  public async getData(): Promise<IFeodoTrackerData[]> {
-    const response = await plugins.nodeFetch(FeodoTracker.FEODO_TRACKER_API_URL, {
-      ...(helpers.findProxy() ? { 
-        agent: helpers.getAgent(),
-      } : {})
-    });
-
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-
-    const data: IFeodoTrackerData[] = await response.json() as IFeodoTrackerData[];
-
-    // Ensure the data is an array and has the expected structure
-    if (!Array.isArray(data) || !data.every(item => 
-      typeof item.ip_address === 'string' &&
-      typeof item.port === 'number' &&
-      (typeof item.hostname === 'string' || item.hostname === null) &&
-      typeof item.as_number === 'number' &&
-      typeof item.as_name === 'string' &&
-      typeof item.country === 'string' &&
-      typeof item.first_seen === 'string' &&
-      typeof item.last_online === 'string' &&
-      typeof item.malware === 'string'
-    )) {
-      throw new Error(`Invalid data structure!`);
-    }
-
-    return data;
-  }
-}
-

ts/abuse.ch.classes.threatfox.ts

@@ -1,81 +0,0 @@
-import * as plugins from './plugins.js';
-import * as paths from './paths.js';
-import * as helpers from './helpers.js';
-
-export interface IThreatFoxData {
-  ID: string;
-  Dateadded: string;
-  URL: string;
-  URLStatus: string;
-  Threat: string;
-  AssociatedTags: string;
-  ThreatFoxLink: string;
-  Reporter: string;
-}
-
-export class ThreatFox {
-  private static readonly THREATFOX_API_URL: string = 'https://threatfox.abuse.ch/export/csv/full'; // Replace with actual API endpoint
-
-  public async getData(): Promise<IThreatFoxData[]> {
-    plugins.smartfile.fs.ensureDirSync(paths.threatFoxTmp);
-    const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
-    const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');
-
-    const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
-      ...(helpers.findProxy() ? { 
-        agent: helpers.getAgent(),
-      } : {})
-    });
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-
-    await new Promise((resolve, reject) => {
-      const fileStream = plugins.fs.createWriteStream(zipPath);
-      // @ts-ignore
-      const readable = plugins.stream.Readable.from(response.body);
-      plugins.stream.pipeline(readable, fileStream, (err) => {
-        if (err) reject(err);
-        else resolve(null);
-      });
-    });
-
-    await new Promise((resolve, reject) => {
-      plugins.stream.pipeline(
-        plugins.fs.createReadStream(zipPath),
-        plugins.unzipper.Extract({ path: paths.threatFoxTmp }),
-        (err) => {
-          if (err) reject(err);
-          else resolve(null);
-        }
-      );
-    });
-
-    let data: IThreatFoxData[] = [];
-    await new Promise((resolve, reject) => {
-      plugins.stream.pipeline(
-        plugins.fs.createReadStream(csvPath),
-        plugins.csv({
-          headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'ThreatFoxLink', 'Reporter'],
-          mapValues: ({ header, value }) => value.trim()
-        }),
-        (err) => {
-          if (err) reject(err);
-        }
-      )
-        .on('data', (row) => {
-          data.push(row);
-        })
-        .on('end', resolve)
-        .on('error', reject);
-    });
-    data = data.map((item) => {
-      return {
-        ...item,
-        URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
-      }
-    });
-    plugins.smartfile.fs.removeSync(paths.threatFoxTmp);
-    return data;
-  }
-}

ts/abuse.ch.classes.urlhouse.ts

@@ -1,81 +0,0 @@
-import * as plugins from './plugins.js';
-import * as paths from './paths.js';
-import * as helpers from './helpers.js';
-
-export interface IUrlHouseData {
-  ID: string;
-  Dateadded: string;
-  URL: string;
-  URLStatus: string;
-  Threat: string;
-  AssociatedTags: string;
-  UrlHausLink: string;
-  Reporter: string;
-}
-
-export class UrlHouse {
-  private static readonly URLHOUSE_API_URL: string = 'https://urlhaus.abuse.ch/downloads/csv/';
-
-  public async getData(): Promise<IUrlHouseData[]> {
-    plugins.smartfile.fs.ensureDirSync(paths.urlHouseTmp);
-    const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
-    const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
-
-    const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
-      ...(helpers.findProxy() ? { 
-        agent: helpers.getAgent(),
-      } : {})
-    });
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-
-    await new Promise((resolve, reject) => {
-      const fileStream = plugins.fs.createWriteStream(zipPath);
-      // @ts-ignore
-      const readable = plugins.stream.Readable.from(response.body);
-      plugins.stream.pipeline(readable, fileStream, (err) => {
-        if (err) reject(err);
-        else resolve(null);
-      });
-    });
-
-    await new Promise((resolve, reject) => {
-      plugins.stream.pipeline(
-        plugins.fs.createReadStream(zipPath),
-        plugins.unzipper.Extract({ path: paths.urlHouseTmp }),
-        (err) => {
-          if (err) reject(err);
-          else resolve(null);
-        }
-      );
-    });
-
-    let data: IUrlHouseData[] = [];
-    await new Promise((resolve, reject) => {
-      plugins.stream.pipeline(
-        plugins.fs.createReadStream(csvPath),
-        plugins.csv({
-          headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'UrlHausLink', 'Reporter'],
-          mapValues: ({ header, value }) => value.trim()
-        }),
-        (err) => {
-          if (err) reject(err);
-        }
-      )
-        .on('data', (row) => {
-          data.push(row);
-        })
-        .on('end', resolve)
-        .on('error', reject);
-    });
-    data = data.map((item) => {
-      return {
-        ...item,
-        URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
-      }
-    });
-    plugins.smartfile.fs.removeSync(paths.urlHouseTmp);
-    return data;
-  }
-}

ts/helpers.ts

@@ -1,8 +0,0 @@
-import * as plugins from './plugins.js';
-
-export const findProxy = () => {
-  return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
-}
-export const getAgent = () => {
-  return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
-}

ts/index.ts

@@ -1,3 +1,3 @@
-export * from './abuse.ch.classes.feodotracker.js';
+import * as plugins from './abuse.ch.plugins.js';
-export * from './abuse.ch.classes.threatfox.js';
+
-export * from './abuse.ch.classes.urlhouse.js';
+export let demoExport = 'Hi there! :) This is an exported string';

ts/paths.ts

@@ -1,8 +0,0 @@
-import * as plugins from './plugins.js';
-
-export const packageDir = plugins.path.join(plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../');
-export const nogitDir = plugins.path.join(packageDir, '.nogit');
-
-export const urlHouseTmp = plugins.path.join(nogitDir, 'tmp.urlhaus');
-export const threatFoxTmp = plugins.path.join(nogitDir, 'tmp.threatfox');
-export const feodoTrackerTmp = plugins.path.join(nogitDir, 'tmp.feodotracker');

ts/plugins.ts

@@ -1,34 +0,0 @@
-// node native
-import * as stream from 'stream';
-import * as util from 'util';
-import unzipper from 'unzipper';
-import csv from 'csv-parser';
-import * as fs from 'fs';
-import * as path from 'path';
-
-export {
-  stream,
-  util,
-  unzipper,
-  csv,
-  fs,
-  path,
-}
-
-// @push.rocks scope
-import * as smartfile from '@push.rocks/smartfile';
-import * as smartpath from '@push.rocks/smartpath';
-
-export {
-  smartfile,
-  smartpath,
-}
-
-// third party
-import nodeFetch from 'node-fetch';
-import * as httpsProxy from 'https-proxy-agent';
-
-export {
-  nodeFetch,
-  httpsProxy,
-}