From 792224c96b70f5989bd9ac3972f5d236ac9d10ac Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Fri, 28 Jul 2023 14:13:16 +0200 Subject: [PATCH] fix(core): update --- package.json | 3 +- pnpm-lock.yaml | 88 +++++++++++++++++++++++--------- ts/00_commitinfo_data.ts | 2 +- ts/abuse.ch.classes.threatfox.ts | 22 +++++--- ts/abuse.ch.classes.urlhouse.ts | 20 ++++++-- ts/helpers.ts | 8 +++ ts/plugins.ts | 9 ++++ 7 files changed, 116 insertions(+), 36 deletions(-) create mode 100644 ts/helpers.ts diff --git a/package.json b/package.json index aa8dd8c..9ea1254 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,9 @@ "dependencies": { "@push.rocks/smartfile": "^10.0.28", "@push.rocks/smartpath": "^5.0.5", - "axios": "^1.4.0", "csv-parser": "^3.0.0", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", "unzipper": "^0.10.14" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 22999f3..778b301 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,12 +11,15 @@ dependencies: '@push.rocks/smartpath': specifier: ^5.0.5 version: 5.0.11 - axios: - specifier: ^1.4.0 - version: 1.4.0 csv-parser: specifier: ^3.0.0 version: 3.0.0 + https-proxy-agent: + specifier: ^7.0.1 + version: 7.0.1 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 unzipper: specifier: ^0.10.14 version: 0.10.14 @@ -1910,6 +1913,15 @@ packages: - supports-color dev: true + /agent-base@7.1.0: + resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==} + engines: {node: '>= 14'} + dependencies: + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + /agentkeepalive@4.3.0: resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==} engines: {node: '>= 8.0.0'} @@ -2012,16 +2024,6 @@ packages: engines: {node: '>=4'} dev: true - /axios@1.4.0: - resolution: {integrity: sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==} - dependencies: - follow-redirects: 1.15.2 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - dev: false - /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2412,6 +2414,11 @@ packages: minimist: 1.2.8 dev: false + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + /dayjs@1.11.9: resolution: {integrity: sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==} @@ -2818,6 +2825,14 @@ packages: xml-js: 1.6.11 dev: true + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + /figures@5.0.0: resolution: {integrity: sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==} engines: {node: '>=14'} @@ -2856,16 +2871,6 @@ packages: path-exists: 4.0.0 dev: true - /follow-redirects@1.15.2: - resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - dev: false - /foreground-child@2.0.0: resolution: {integrity: sha512-dCIq9FpEcyQyXKCkyzmlPTFNgrCzPudOe+mhvJU5zAtlBnGVy2yKxtfsxK2tQBThwq225jcvBjpw1Gr40uzZCA==} engines: {node: '>=8.0.0'} @@ -2894,6 +2899,13 @@ packages: combined-stream: 1.0.8 mime-types: 2.1.35 + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -3147,6 +3159,16 @@ packages: - supports-color dev: true + /https-proxy-agent@7.0.1: + resolution: {integrity: sha512-Eun8zV0kcYS1g19r78osiQLEFIRspRUDd9tIfBCTBPBeMieF/EsJNL8VI3xOIdYRDEkjQnqOYPsZ2DsWsVsFwQ==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.0 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + /humanize-ms@1.2.1: resolution: {integrity: sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=} dependencies: @@ -3795,6 +3817,11 @@ packages: lower-case: 1.1.4 dev: true + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + /node-fetch@2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} @@ -3807,6 +3834,15 @@ packages: whatwg-url: 5.0.0 dev: true + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + /normalize-newline@4.1.0: resolution: {integrity: sha512-ff4jKqMI8Xl50/4Mms/9jPobzAV/UK+kXG2XJ/7AqOmxIx8mqfqTIHYxuAnEgJ2AQeBbLnlbmZ5+38Y9A0w/YA==} engines: {node: '>=12'} @@ -4045,6 +4081,7 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true /public-ip@6.0.1: resolution: {integrity: sha512-1/Mxa1MKrAQ4jF5IalECSBtB0W1FAtnG+9c5X16jjvV/Gx9fiRy7xXIrHlBGYjnTlai0zdZkM3LrpmASavmAEg==} @@ -4819,6 +4856,11 @@ packages: resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==} dev: true + /web-streams-polyfill@3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false + /webidl-conversions@3.0.1: resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=} dev: true diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 240372d..031ec20 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@apiclient.xyz/abuse.ch', - version: '1.0.6', + version: '1.0.7', description: 'an unofficial client to retrieve abuse.ch data' } diff --git a/ts/abuse.ch.classes.threatfox.ts b/ts/abuse.ch.classes.threatfox.ts index 2492e28..f0bd7a4 100644 --- a/ts/abuse.ch.classes.threatfox.ts +++ b/ts/abuse.ch.classes.threatfox.ts @@ -1,6 +1,6 @@ import * as plugins from './plugins.js'; import * as paths from './paths.js'; -import axios from 'axios'; +import * as helpers from './helpers.js'; export interface IThreatFoxData { ID: string; @@ -21,13 +21,23 @@ export class ThreatFox { const zipPath = plugins.path.join(paths.threatFoxTmp, 'threatfox.zip'); const csvPath = plugins.path.join(paths.threatFoxTmp, 'full.csv'); - const response = await axios.get(ThreatFox.THREATFOX_API_URL, { responseType: 'stream' }); - + const response = await plugins.nodeFetch(ThreatFox.THREATFOX_API_URL, { + ...(helpers.findProxy() ? { + agent: helpers.getAgent(), + } : {}) + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + await new Promise((resolve, reject) => { const fileStream = plugins.fs.createWriteStream(zipPath); - response.data.pipe(fileStream); - fileStream.on('finish', resolve); - fileStream.on('error', reject); + // @ts-ignore + const readable = plugins.stream.Readable.from(response.body); + plugins.stream.pipeline(readable, fileStream, (err) => { + if (err) reject(err); + else resolve(null); + }); }); await new Promise((resolve, reject) => { diff --git a/ts/abuse.ch.classes.urlhouse.ts b/ts/abuse.ch.classes.urlhouse.ts index 031781c..bee9227 100644 --- a/ts/abuse.ch.classes.urlhouse.ts +++ b/ts/abuse.ch.classes.urlhouse.ts @@ -1,6 +1,6 @@ import * as plugins from './plugins.js'; import * as paths from './paths.js'; -import axios from 'axios'; +import * as helpers from './helpers.js'; export interface IUrlHouseData { ID: string; @@ -21,13 +21,23 @@ export class UrlHouse { const zipPath = plugins.path.join(paths.urlHouseTmp, 'urlhaus.zip'); const csvPath = plugins.path.join(paths.urlHouseTmp, 'csv.txt'); - const response = await axios.get(UrlHouse.URLHOUSE_API_URL, { responseType: 'stream' }); + const response = await plugins.nodeFetch(UrlHouse.URLHOUSE_API_URL, { + ...(helpers.findProxy() ? { + agent: helpers.getAgent(), + } : {}) + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } await new Promise((resolve, reject) => { const fileStream = plugins.fs.createWriteStream(zipPath); - response.data.pipe(fileStream); - fileStream.on('finish', resolve); - fileStream.on('error', reject); + // @ts-ignore + const readable = plugins.stream.Readable.from(response.body); + plugins.stream.pipeline(readable, fileStream, (err) => { + if (err) reject(err); + else resolve(null); + }); }); await new Promise((resolve, reject) => { diff --git a/ts/helpers.ts b/ts/helpers.ts new file mode 100644 index 0000000..aff0f24 --- /dev/null +++ b/ts/helpers.ts @@ -0,0 +1,8 @@ +import * as plugins from './plugins.js'; + +export const findProxy = () => { + return process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY || process.env.http_proxy +} +export const getAgent = () => { + return new plugins.httpsProxy.HttpsProxyAgent(findProxy()); +} \ No newline at end of file diff --git a/ts/plugins.ts b/ts/plugins.ts index b7269ac..5a8b681 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -22,4 +22,13 @@ import * as smartpath from '@push.rocks/smartpath'; export { smartfile, smartpath, +} + +// third party +import nodeFetch from 'node-fetch'; +import * as httpsProxy from 'https-proxy-agent'; + +export { + nodeFetch, + httpsProxy, } \ No newline at end of file