2023-07-28 05:31:59 +00:00
|
|
|
import * as plugins from './plugins.js';
|
|
|
|
import * as paths from './paths.js';
|
2023-07-28 07:54:22 +00:00
|
|
|
import axios from 'axios';
|
2023-07-28 05:31:59 +00:00
|
|
|
|
|
|
|
export interface IUrlHouseData {
|
|
|
|
ID: string;
|
|
|
|
Dateadded: string;
|
|
|
|
URL: string;
|
|
|
|
URLStatus: string;
|
|
|
|
Threat: string;
|
|
|
|
AssociatedTags: string;
|
|
|
|
UrlHausLink: string;
|
|
|
|
Reporter: string;
|
|
|
|
}
|
|
|
|
|
|
|
|
export class UrlHouse {
|
|
|
|
private static readonly URLHOUSE_API_URL: string = 'https://urlhaus.abuse.ch/downloads/csv/';
|
|
|
|
|
|
|
|
public async getData(): Promise<IUrlHouseData[]> {
|
|
|
|
plugins.smartfile.fs.ensureDirSync(paths.urlHouseTmp);
|
|
|
|
const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip');
|
|
|
|
const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt');
|
|
|
|
|
2023-07-28 07:54:22 +00:00
|
|
|
const response = await axios.get(UrlHouse.URLHOUSE_API_URL, { responseType: 'stream' });
|
2023-07-28 05:31:59 +00:00
|
|
|
|
|
|
|
await new Promise((resolve, reject) => {
|
|
|
|
const fileStream = plugins.fs.createWriteStream(zipPath);
|
2023-07-28 07:54:22 +00:00
|
|
|
response.data.pipe(fileStream);
|
|
|
|
fileStream.on('finish', resolve);
|
|
|
|
fileStream.on('error', reject);
|
2023-07-28 05:31:59 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
await new Promise((resolve, reject) => {
|
|
|
|
plugins.stream.pipeline(
|
|
|
|
plugins.fs.createReadStream(zipPath),
|
|
|
|
plugins.unzipper.Extract({ path: paths.urlHouseTmp }),
|
|
|
|
(err) => {
|
|
|
|
if (err) reject(err);
|
|
|
|
else resolve(null);
|
|
|
|
}
|
|
|
|
);
|
|
|
|
});
|
|
|
|
|
|
|
|
let data: IUrlHouseData[] = [];
|
|
|
|
await new Promise((resolve, reject) => {
|
|
|
|
plugins.stream.pipeline(
|
|
|
|
plugins.fs.createReadStream(csvPath),
|
|
|
|
plugins.csv({
|
|
|
|
headers: ['ID', 'Dateadded', 'URL', 'URLStatus', 'Threat', 'AssociatedTags', 'UrlHausLink', 'Reporter'],
|
|
|
|
mapValues: ({ header, value }) => value.trim()
|
|
|
|
}),
|
|
|
|
(err) => {
|
|
|
|
if (err) reject(err);
|
|
|
|
}
|
|
|
|
)
|
|
|
|
.on('data', (row) => {
|
|
|
|
data.push(row);
|
|
|
|
})
|
|
|
|
.on('end', resolve)
|
|
|
|
.on('error', reject);
|
|
|
|
});
|
|
|
|
data = data.map((item) => {
|
|
|
|
return {
|
|
|
|
...item,
|
|
|
|
URL: item.URL?.replace('http', 'ht-NOCLICK-NOLINK-tp'),
|
|
|
|
}
|
|
|
|
});
|
|
|
|
plugins.smartfile.fs.removeSync(paths.urlHouseTmp);
|
|
|
|
return data;
|
|
|
|
}
|
|
|
|
}
|