From ca1d0bd9a3797a48d710dd362be48272d162f965 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Fri, 28 Jul 2023 09:54:22 +0200 Subject: [PATCH] fix(core): update --- package.json | 1 + pnpm-lock.yaml | 24 +++++++++++++++++++++++- ts/00_commitinfo_data.ts | 2 +- ts/abuse.ch.classes.threatfox.ts | 17 ++++++----------- ts/abuse.ch.classes.urlhouse.ts | 15 +++++---------- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 9c6e939..2795b64 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "dependencies": { "@push.rocks/smartfile": "^10.0.28", "@push.rocks/smartpath": "^5.0.5", + "axios": "^1.4.0", "csv-parser": "^3.0.0", "unzipper": "^0.10.14" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5048dc1..22999f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ dependencies: '@push.rocks/smartpath': specifier: ^5.0.5 version: 5.0.11 + axios: + specifier: ^1.4.0 + version: 1.4.0 csv-parser: specifier: ^3.0.0 version: 3.0.0 @@ -2009,6 +2012,16 @@ packages: engines: {node: '>=4'} dev: true + /axios@1.4.0: + resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} + dependencies: + follow-redirects: 1.15.2 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2843,6 +2856,16 @@ packages: path-exists: 4.0.0 dev: true + /follow-redirects@1.15.2: + resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + /foreground-child@2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} engines: {node: '>=8.0.0'} @@ -4022,7 +4045,6 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: true /public-ip@6.0.1: resolution: {integrity: sha512-1/Mxa1MKrAQ4jF5IalECSBtB0W1FAtnG+9c5X16jjvV/Gx9fiRy7xXIrHlBGYjnTlai0zdZkM3LrpmASavmAEg==} diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 8256467..b9368e4 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/abuse.ch', - version: '1.0.4', + version: '1.0.5', description: 'an unofficial client to retrieve abuse.ch data' } diff --git a/ts/abuse.ch.classes.threatfox.ts b/ts/abuse.ch.classes.threatfox.ts index 8d27335..2492e28 100644 --- a/ts/abuse.ch.classes.threatfox.ts +++ b/ts/abuse.ch.classes.threatfox.ts @@ -1,5 +1,6 @@ import * as plugins from './plugins.js'; import * as paths from './paths.js'; +import axios from 'axios'; export interface IThreatFoxData { ID: string; @@ -20,19 +21,13 @@ export class ThreatFox { const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip'); const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv'); - const response = await fetch(ThreatFox.THREATFOX_API_URL); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - + const response = await axios.get(ThreatFox.THREATFOX_API_URL, { responseType: 'stream' }); + await new Promise((resolve, reject) => { const fileStream = plugins.fs.createWriteStream(zipPath); - // @ts-ignore - const readable = plugins.stream.Readable.from(response.body); - plugins.stream.pipeline(readable, fileStream, (err) => { - if (err) reject(err); - else resolve(null); - }); + response.data.pipe(fileStream); + fileStream.on('finish', resolve); + fileStream.on('error', reject); }); await new Promise((resolve, reject) => { diff --git a/ts/abuse.ch.classes.urlhouse.ts b/ts/abuse.ch.classes.urlhouse.ts index 0651212..031781c 100644 --- a/ts/abuse.ch.classes.urlhouse.ts +++ b/ts/abuse.ch.classes.urlhouse.ts @@ -1,5 +1,6 @@ import * as plugins from './plugins.js'; import * as paths from './paths.js'; +import axios from 'axios'; export interface IUrlHouseData { ID: string; @@ -20,19 +21,13 @@ export class UrlHouse { const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip'); const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt'); - const response = await fetch(UrlHouse.URLHOUSE_API_URL); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } + const response = await axios.get(UrlHouse.URLHOUSE_API_URL, { responseType: 'stream' }); await new Promise((resolve, reject) => { const fileStream = plugins.fs.createWriteStream(zipPath); - // @ts-ignore - const readable = plugins.stream.Readable.from(response.body); - plugins.stream.pipeline(readable, fileStream, (err) => { - if (err) reject(err); - else resolve(null); - }); + response.data.pipe(fileStream); + fileStream.on('finish', resolve); + fileStream.on('error', reject); }); await new Promise((resolve, reject) => {