Compare commits
No commits in common. "master" and "v1.0.2" have entirely different histories.
12
package.json
12
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@apiclient.xyz/abuse.ch",
|
||||
"version": "1.0.12",
|
||||
"version": "1.0.2",
|
||||
"private": false,
|
||||
"description": "an unofficial client to retrieve abuse.ch data",
|
||||
"main": "dist_ts/index.js",
|
||||
@ -36,13 +36,5 @@
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
],
|
||||
"dependencies": {
|
||||
"@push.rocks/smartfile": "^10.0.28",
|
||||
"@push.rocks/smartpath": "^5.0.5",
|
||||
"csv-parser": "^3.0.0",
|
||||
"https-proxy-agent": "^7.0.1",
|
||||
"node-fetch": "^3.3.2",
|
||||
"unzipper": "^0.10.14"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
326
pnpm-lock.yaml
generated
326
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
19
test/test.ts
19
test/test.ts
@ -1,23 +1,8 @@
|
||||
import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
|
||||
import * as abuseCh from '../ts/index.js';
|
||||
|
||||
tap.test('should deal with UrlHouse data', async () => {
|
||||
const urlHouse = new abuseCh.UrlHouse();
|
||||
const data = await urlHouse.getData();
|
||||
console.log(data.length);
|
||||
});
|
||||
|
||||
tap.test('should deal with UrlHouse data', async () => {
|
||||
const threatFox = new abuseCh.ThreatFox();
|
||||
const data = await threatFox.getData();
|
||||
console.log(data.length);
|
||||
});
|
||||
|
||||
tap.test('should deal with FeodoTracker data', async () => {
|
||||
const feodoTracker = new abuseCh.FeodoTracker();
|
||||
const data = await feodoTracker.getData();
|
||||
console.log(data.length);
|
||||
console.log(data[1]);
|
||||
tap.test('first test', async () => {
|
||||
console.log(abuseCh);
|
||||
});
|
||||
|
||||
tap.start();
|
||||
|
||||
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@apiclient.xyz/abuse.ch',
|
||||
version: '1.0.12',
|
||||
version: '1.0.2',
|
||||
description: 'an unofficial client to retrieve abuse.ch data'
|
||||
}
|
||||
|
||||
@ -1,52 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import * as helpers from './helpers.js';
|
||||
|
||||
export interface IFeodoTrackerData {
|
||||
ip_address: string;
|
||||
port: number;
|
||||
status: string;
|
||||
hostname: string | null;
|
||||
as_number: number;
|
||||
as_name: string;
|
||||
country: string;
|
||||
first_seen: string;
|
||||
last_online: string;
|
||||
malware: string;
|
||||
}
|
||||
|
||||
export class FeodoTracker {
|
||||
private static readonly FEODO_TRACKER_API_URL: string = 'https://feodotracker.abuse.ch/downloads/ipblocklist.json';
|
||||
|
||||
public async getData(): Promise<IFeodoTrackerData[]> {
|
||||
const response = await plugins.nodeFetch(FeodoTracker.FEODO_TRACKER_API_URL, {
|
||||
...(helpers.findProxy() ? {
|
||||
agent: helpers.getAgent(),
|
||||
} : {})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: IFeodoTrackerData[] = await response.json() as IFeodoTrackerData[];
|
||||
|
||||
// Ensure the data is an array and has the expected structure
|
||||
if (!Array.isArray(data) || !data.every(item =>
|
||||
typeof item.ip_address === 'string' &&
|
||||
typeof item.port === 'number' &&
|
||||
(typeof item.hostname === 'string' || item.hostname === null) &&
|
||||
typeof item.as_number === 'number' &&
|
||||
typeof item.as_name === 'string' &&
|
||||
typeof item.country === 'string' &&
|
||||
typeof item.first_seen === 'string' &&
|
||||
typeof item.last_online === 'string' &&
|
||||
typeof item.malware === 'string'
|
||||
)) {
|
||||
throw new Error(`Invalid data structure!`);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,81 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import * as helpers from './helpers.js';
|
||||
|
||||
export interface IThreatFoxData {
|
||||
ID: string;
|
||||
Dateadded: string;
|
||||
URL: string;
|
||||
URLStatus: string;
|
||||
Threat: string;
|
||||
AssociatedTags: string;
|
||||
ThreatFoxLink: string;
|
||||
Reporter: string;
|
||||
}
|
||||
|
||||
export class ThreatFox {
|
||||
private static readonly THREATFOX_API_URL: string = 'https://threatfox.abuse.ch/export/csv/full'; // Replace with actual API endpoint
|
||||
|
||||
public async getData(): Promise<IThreatFoxData[]> {
|
||||
plugins.smartfile.fs.ensureDirSync(paths.threatFoxTmp);
|
||||
const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip');
|
||||
const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv');
|
||||
|
||||
const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, {
|
||||
...(helpers.findProxy() ? {
|
||||
agent: helpers.getAgent(),
|
||||
} : {})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const fileStream = plugins.fs.createWriteStream(zipPath);
|
||||
// @ts-ignore
|
||||
const readable = plugins.stream.Readable.from(response.body);
|
||||
plugins.stream.pipeline(readable, fileStream, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve(null);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
plugins.stream.pipeline(
|
||||
plugins.fs.createReadStream(zipPath),
|
||||
plugins.unzipper.Extract({ path: paths.threatFoxTmp }),
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
else resolve(null);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
let data: IThreatFoxData[] = [];
|
||||
await new Promise((resolve, reject) => {
|
||||
plugins.stream.pipeline(
|
||||
plugins.fs.createReadStream(csvPath),
|
||||
plugins.csv({
|
||||
headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'ThreatFoxLink', 'Reporter'],
|
||||
mapValues: ({ header, value }) => value.trim()
|
||||
}),
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
}
|
||||
)
|
||||
.on('data', (row) => {
|
||||
data.push(row);
|
||||
})
|
||||
.on('end', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
data = data.map((item) => {
|
||||
return {
|
||||
...item,
|
||||
URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
|
||||
}
|
||||
});
|
||||
plugins.smartfile.fs.removeSync(paths.threatFoxTmp);
|
||||
return data;
|
||||
}
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import * as helpers from './helpers.js';
|
||||
|
||||
export interface IUrlHouseData {
|
||||
ID: string;
|
||||
Dateadded: string;
|
||||
URL: string;
|
||||
URLStatus: string;
|
||||
Threat: string;
|
||||
AssociatedTags: string;
|
||||
UrlHausLink: string;
|
||||
Reporter: string;
|
||||
}
|
||||
|
||||
export class UrlHouse {
|
||||
private static readonly URLHOUSE_API_URL: string = 'https://urlhaus.abuse.ch/downloads/csv/';
|
||||
|
||||
public async getData(): Promise<IUrlHouseData[]> {
|
||||
plugins.smartfile.fs.ensureDirSync(paths.urlHouseTmp);
|
||||
const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
|
||||
const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
|
||||
|
||||
const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
|
||||
...(helpers.findProxy() ? {
|
||||
agent: helpers.getAgent(),
|
||||
} : {})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const fileStream = plugins.fs.createWriteStream(zipPath);
|
||||
// @ts-ignore
|
||||
const readable = plugins.stream.Readable.from(response.body);
|
||||
plugins.stream.pipeline(readable, fileStream, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve(null);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
plugins.stream.pipeline(
|
||||
plugins.fs.createReadStream(zipPath),
|
||||
plugins.unzipper.Extract({ path: paths.urlHouseTmp }),
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
else resolve(null);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
let data: IUrlHouseData[] = [];
|
||||
await new Promise((resolve, reject) => {
|
||||
plugins.stream.pipeline(
|
||||
plugins.fs.createReadStream(csvPath),
|
||||
plugins.csv({
|
||||
headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'UrlHausLink', 'Reporter'],
|
||||
mapValues: ({ header, value }) => value.trim()
|
||||
}),
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
}
|
||||
)
|
||||
.on('data', (row) => {
|
||||
data.push(row);
|
||||
})
|
||||
.on('end', resolve)
|
||||
.on('error', reject);
|
||||
});
|
||||
data = data.map((item) => {
|
||||
return {
|
||||
...item,
|
||||
URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
|
||||
}
|
||||
});
|
||||
plugins.smartfile.fs.removeSync(paths.urlHouseTmp);
|
||||
return data;
|
||||
}
|
||||
}
|
||||
0
ts/abuse.ch.plugins.ts
Normal file
0
ts/abuse.ch.plugins.ts
Normal file
@ -1,8 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export const findProxy = () => {
|
||||
return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy
|
||||
}
|
||||
export const getAgent = () => {
|
||||
return new plugins.httpsProxy.HttpsProxyAgent(findProxy());
|
||||
}
|
||||
@ -1,3 +1,3 @@
|
||||
export * from './abuse.ch.classes.feodotracker.js';
|
||||
export * from './abuse.ch.classes.threatfox.js';
|
||||
export * from './abuse.ch.classes.urlhouse.js';
|
||||
import * as plugins from './abuse.ch.plugins.js';
|
||||
|
||||
export let demoExport = 'Hi there! :) This is an exported string';
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export const packageDir = plugins.path.join(plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), '../');
|
||||
export const nogitDir = plugins.path.join(packageDir, '.nogit');
|
||||
|
||||
export const urlHouseTmp = plugins.path.join(nogitDir, 'tmp.urlhaus');
|
||||
export const threatFoxTmp = plugins.path.join(nogitDir, 'tmp.threatfox');
|
||||
export const feodoTrackerTmp = plugins.path.join(nogitDir, 'tmp.feodotracker');
|
||||
@ -1,34 +0,0 @@
|
||||
// node native
|
||||
import * as stream from 'stream';
|
||||
import * as util from 'util';
|
||||
import unzipper from 'unzipper';
|
||||
import csv from 'csv-parser';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export {
|
||||
stream,
|
||||
util,
|
||||
unzipper,
|
||||
csv,
|
||||
fs,
|
||||
path,
|
||||
}
|
||||
|
||||
// @push.rocks scope
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
|
||||
export {
|
||||
smartfile,
|
||||
smartpath,
|
||||
}
|
||||
|
||||
// third party
|
||||
import nodeFetch from 'node-fetch';
|
||||
import * as httpsProxy from 'https-proxy-agent';
|
||||
|
||||
export {
|
||||
nodeFetch,
|
||||
httpsProxy,
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user