82 lines
2.3 KiB
TypeScript
82 lines
2.3 KiB
TypeScript
import * as plugins from './plugins.js';
|
|
import * as paths from './paths.js';
|
|
import * as helpers from './helpers.js';
|
|
|
|
export interface IUrlHouseData {
|
|
ID: string;
|
|
Dateadded: string;
|
|
URL: string;
|
|
URLStatus: string;
|
|
Threat: string;
|
|
AssociatedTags: string;
|
|
UrlHausLink: string;
|
|
Reporter: string;
|
|
}
|
|
|
|
export class UrlHouse {
|
|
private static readonly URLHOUSE_API_URL: string = 'https://urlhaus.abuse.ch/downloads/csv/';
|
|
|
|
public async getData(): Promise<IUrlHouseData[]> {
|
|
plugins.smartfile.fs.ensureDirSync(paths.urlHouseTmp);
|
|
const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
|
|
const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
|
|
|
|
const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, {
|
|
...(helpers.findProxy() ? {
|
|
agent: helpers.getAgent(),
|
|
} : {})
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP error! status: ${response.status}`);
|
|
}
|
|
|
|
await new Promise((resolve, reject) => {
|
|
const fileStream = plugins.fs.createWriteStream(zipPath);
|
|
// @ts-ignore
|
|
const readable = plugins.stream.Readable.from(response.body);
|
|
plugins.stream.pipeline(readable, fileStream, (err) => {
|
|
if (err) reject(err);
|
|
else resolve(null);
|
|
});
|
|
});
|
|
|
|
await new Promise((resolve, reject) => {
|
|
plugins.stream.pipeline(
|
|
plugins.fs.createReadStream(zipPath),
|
|
plugins.unzipper.Extract({ path: paths.urlHouseTmp }),
|
|
(err) => {
|
|
if (err) reject(err);
|
|
else resolve(null);
|
|
}
|
|
);
|
|
});
|
|
|
|
let data: IUrlHouseData[] = [];
|
|
await new Promise((resolve, reject) => {
|
|
plugins.stream.pipeline(
|
|
plugins.fs.createReadStream(csvPath),
|
|
plugins.csv({
|
|
headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'UrlHausLink', 'Reporter'],
|
|
mapValues: ({ header, value }) => value.trim()
|
|
}),
|
|
(err) => {
|
|
if (err) reject(err);
|
|
}
|
|
)
|
|
.on('data', (row) => {
|
|
data.push(row);
|
|
})
|
|
.on('end', resolve)
|
|
.on('error', reject);
|
|
});
|
|
data = data.map((item) => {
|
|
return {
|
|
...item,
|
|
URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
|
|
}
|
|
});
|
|
plugins.smartfile.fs.removeSync(paths.urlHouseTmp);
|
|
return data;
|
|
}
|
|
}
|