import * as plugins from './webrequest.plugins.js'; /** * web request */ export class WebRequest { private static polyfillStatusEvaluated = false; private static neededPolyfillsLoadedDeferred = plugins.smartpromise.defer(); public static async loadNeededPolyfills() { if (this.polyfillStatusEvaluated) { return this.neededPolyfillsLoadedDeferred.promise; } this.polyfillStatusEvaluated = true; const smartenv = new plugins.smartenv.Smartenv(); if (!smartenv.isBrowser) { this.polyfillStatusEvaluated = true; const fetchMod = await smartenv.getSafeNodeModule('node-fetch'); globalThis.Response = fetchMod.Response; globalThis.fetch = fetchMod.default; } this.neededPolyfillsLoadedDeferred.resolve(); } public cacheStore = new plugins.webstore.WebStore({ dbName: 'webrequest', storeName: 'webrequest', }); constructor() { WebRequest.loadNeededPolyfills(); } public async getJson(urlArg: string, useCacheArg: boolean = false) { await WebRequest.neededPolyfillsLoadedDeferred.promise; const response: Response = await this.request(urlArg, { method: 'GET', useCache: useCacheArg, }); const responseText = await response.text(); const responseResult = plugins.smartjson.parse(responseText); return responseResult; } /** * postJson */ public async postJson(urlArg: string, requestBody?: any, useStoreAsFallback: boolean = false) { await WebRequest.neededPolyfillsLoadedDeferred.promise; const response: Response = await this.request(urlArg, { body: plugins.smartjson.stringify(requestBody), method: 'POST', }); const responseText = await response.text(); const responseResult = plugins.smartjson.parse(responseText); return responseResult; } /** * put js */ public async putJson(urlArg: string, requestBody?: any, useStoreAsFallback: boolean = false) { await WebRequest.neededPolyfillsLoadedDeferred.promise; const response: Response = await this.request(urlArg, { body: plugins.smartjson.stringify(requestBody), method: 'PUT', }); const responseText = await response.text(); const responseResult = plugins.smartjson.parse(responseText); return responseResult; } /** * put js */ public async deleteJson(urlArg: string, useStoreAsFallback: boolean = false) { await WebRequest.neededPolyfillsLoadedDeferred.promise; const response: Response = await this.request(urlArg, { method: 'GET', }); const responseText = await response.text(); const responseResult = plugins.smartjson.parse(responseText); return responseResult; } public async request( urlArg: string, optionsArg: { method: 'GET' | 'POST' | 'PUT' | 'DELETE'; body?: any; headers?: HeadersInit; useCache?: boolean; timeoutMs?: number; } ) { await WebRequest.neededPolyfillsLoadedDeferred.promise; const controller = new AbortController(); if (optionsArg.timeoutMs) { plugins.smartdelay.delayFor(optionsArg.timeoutMs).then(() => { controller.abort(); }); } let cachedResponseDeferred = plugins.smartpromise.defer(); let cacheUsed = false; if (optionsArg.useCache && this.cacheStore.check(urlArg)) { const responseBuffer: ArrayBuffer = await this.cacheStore.get(urlArg) cachedResponseDeferred.resolve(new Response(responseBuffer, {})); } else { cachedResponseDeferred.resolve(null); } const response: Response = await fetch(urlArg, { signal: controller.signal, method: optionsArg.method, headers: { 'Content-Type': 'application/json', ...(optionsArg.headers || {}), }, body: optionsArg.body, }).catch(async err => { if (optionsArg.useCache && await cachedResponseDeferred.promise) { cacheUsed = true; const cachedResponse = cachedResponseDeferred.promise; return cachedResponse; } else { return err; } }); if (!cacheUsed && optionsArg.useCache && response.status < 300) { const buffer = await response.clone().arrayBuffer(); await this.cacheStore.set(urlArg, buffer); } console.log(`${urlArg} answers with status: ${response.status}`); return response; } /** * a multi endpoint, fault tolerant request function */ public async requestMultiEndpoint( urlArg: string | string[], optionsArg: { method: 'GET' | 'POST' | 'PUT' | 'DELETE'; body?: any; headers?: HeadersInit; } ): Promise { await WebRequest.neededPolyfillsLoadedDeferred.promise; let allUrls: string[]; let usedUrlIndex = 0; // determine what we got if (Array.isArray(urlArg)) { allUrls = urlArg; } else { allUrls = [urlArg]; } const requestHistory: string[] = []; // keep track of the request history const doHistoryCheck = async ( // check history for a historyEntryTypeArg: string ) => { requestHistory.push(historyEntryTypeArg); if (historyEntryTypeArg === '429') { console.log('got 429, so waiting a little bit.'); await plugins.smartdelay.delayFor(Math.floor(Math.random() * (2000 - 1000 + 1)) + 1000); // wait between 1 and 10 seconds } let numOfHistoryType = 0; for (const entry of requestHistory) { if (entry === historyEntryTypeArg) numOfHistoryType++; } if (numOfHistoryType > 2 * allUrls.length * usedUrlIndex) { usedUrlIndex++; } }; // lets go recursive const doRequest = async (urlToUse: string): Promise => { if (!urlToUse) { throw new Error('request failed permanently'); } console.log(`Getting ${urlToUse} with method ${optionsArg.method}`); const response = await fetch(urlToUse, { method: optionsArg.method, headers: { 'Content-Type': 'application/json', ...(optionsArg.headers || {}), }, body: optionsArg.body, }); console.log(`${urlToUse} answers with status: ${response.status}`); if (response.status >= 200 && response.status < 300) { return response; } else { // lets perform a history check to determine failed urls await doHistoryCheck(response.status.toString()); // lets fire the request const result = await doRequest(allUrls[usedUrlIndex]); return result; } }; const finalResponse: Response = await doRequest(allUrls[usedUrlIndex]); return finalResponse; } }