2022-03-16 15:23:32 +00:00
|
|
|
import * as plugins from './webrequest.plugins.js';
|
2018-11-30 16:12:48 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* web request
|
|
|
|
*/
|
|
|
|
export class WebRequest {
|
2022-03-16 15:23:32 +00:00
|
|
|
private static polyfillStatusEvaluated = false;
|
|
|
|
private static neededPolyfillsLoadedDeferred = plugins.smartpromise.defer();
|
|
|
|
public static async loadNeededPolyfills() {
|
|
|
|
if (this.polyfillStatusEvaluated) {
|
|
|
|
return this.neededPolyfillsLoadedDeferred.promise;
|
|
|
|
}
|
|
|
|
this.polyfillStatusEvaluated = true;
|
2020-06-25 23:34:59 +00:00
|
|
|
const smartenv = new plugins.smartenv.Smartenv();
|
2022-03-16 15:23:32 +00:00
|
|
|
if (!smartenv.isBrowser) {
|
|
|
|
this.polyfillStatusEvaluated = true;
|
2022-05-29 18:22:42 +00:00
|
|
|
const fetchMod = await smartenv.getSafeNodeModule('node-fetch');
|
|
|
|
globalThis.Response = fetchMod.Response;
|
|
|
|
globalThis.fetch = fetchMod.default;
|
2020-06-25 22:45:19 +00:00
|
|
|
}
|
2022-03-16 15:23:32 +00:00
|
|
|
this.neededPolyfillsLoadedDeferred.resolve();
|
2020-06-25 22:45:19 +00:00
|
|
|
}
|
|
|
|
|
2022-05-29 18:22:42 +00:00
|
|
|
public cacheStore = new plugins.webstore.WebStore({
|
|
|
|
dbName: 'webrequest',
|
|
|
|
storeName: 'webrequest',
|
|
|
|
});
|
|
|
|
|
2020-06-25 22:45:19 +00:00
|
|
|
constructor() {
|
|
|
|
WebRequest.loadNeededPolyfills();
|
|
|
|
}
|
|
|
|
|
2022-05-29 18:22:42 +00:00
|
|
|
public async getJson(urlArg: string, useCacheArg: boolean = false) {
|
2022-03-16 15:23:32 +00:00
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
2019-06-04 13:40:30 +00:00
|
|
|
const response: Response = await this.request(urlArg, {
|
2020-10-09 10:14:45 +00:00
|
|
|
method: 'GET',
|
2022-05-29 18:22:42 +00:00
|
|
|
useCache: useCacheArg,
|
2019-06-04 13:40:30 +00:00
|
|
|
});
|
2020-10-09 10:05:26 +00:00
|
|
|
const responseText = await response.text();
|
|
|
|
const responseResult = plugins.smartjson.parse(responseText);
|
|
|
|
return responseResult;
|
2019-06-04 13:40:30 +00:00
|
|
|
}
|
|
|
|
|
2018-11-30 16:12:48 +00:00
|
|
|
/**
|
2019-06-04 13:40:30 +00:00
|
|
|
* postJson
|
2018-11-30 16:12:48 +00:00
|
|
|
*/
|
2022-05-29 18:53:16 +00:00
|
|
|
public async postJson(urlArg: string, requestBody?: any, useCacheArg: boolean = false) {
|
2022-03-16 15:23:32 +00:00
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
2019-06-04 13:40:30 +00:00
|
|
|
const response: Response = await this.request(urlArg, {
|
2020-10-09 10:05:26 +00:00
|
|
|
body: plugins.smartjson.stringify(requestBody),
|
2020-10-09 10:14:45 +00:00
|
|
|
method: 'POST',
|
2022-05-29 19:29:42 +00:00
|
|
|
useCache: useCacheArg,
|
2019-06-04 13:23:47 +00:00
|
|
|
});
|
2020-10-09 10:05:26 +00:00
|
|
|
const responseText = await response.text();
|
|
|
|
const responseResult = plugins.smartjson.parse(responseText);
|
|
|
|
return responseResult;
|
2019-06-04 13:23:47 +00:00
|
|
|
}
|
2018-11-30 16:12:48 +00:00
|
|
|
|
|
|
|
/**
|
2019-06-04 13:40:30 +00:00
|
|
|
* put js
|
2018-11-30 16:12:48 +00:00
|
|
|
*/
|
2022-05-29 18:22:42 +00:00
|
|
|
public async putJson(urlArg: string, requestBody?: any, useStoreAsFallback: boolean = false) {
|
2022-03-16 15:23:32 +00:00
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
2019-06-04 13:40:30 +00:00
|
|
|
const response: Response = await this.request(urlArg, {
|
2020-10-09 10:05:26 +00:00
|
|
|
body: plugins.smartjson.stringify(requestBody),
|
2020-10-09 10:14:45 +00:00
|
|
|
method: 'PUT',
|
2019-06-04 13:40:30 +00:00
|
|
|
});
|
2020-10-09 10:05:26 +00:00
|
|
|
const responseText = await response.text();
|
|
|
|
const responseResult = plugins.smartjson.parse(responseText);
|
|
|
|
return responseResult;
|
2019-06-04 13:40:30 +00:00
|
|
|
}
|
2018-11-30 16:12:48 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* put js
|
|
|
|
*/
|
2022-05-29 18:22:42 +00:00
|
|
|
public async deleteJson(urlArg: string, useStoreAsFallback: boolean = false) {
|
2022-03-16 15:23:32 +00:00
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
2019-06-04 13:40:30 +00:00
|
|
|
const response: Response = await this.request(urlArg, {
|
2020-10-09 10:14:45 +00:00
|
|
|
method: 'GET',
|
2019-06-04 13:40:30 +00:00
|
|
|
});
|
2020-10-09 10:05:26 +00:00
|
|
|
const responseText = await response.text();
|
|
|
|
const responseResult = plugins.smartjson.parse(responseText);
|
|
|
|
return responseResult;
|
2019-06-04 13:40:30 +00:00
|
|
|
}
|
2018-11-30 16:12:48 +00:00
|
|
|
|
2022-05-29 18:22:42 +00:00
|
|
|
public async request(
|
|
|
|
urlArg: string,
|
|
|
|
optionsArg: {
|
|
|
|
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
|
|
|
body?: any;
|
|
|
|
headers?: HeadersInit;
|
|
|
|
useCache?: boolean;
|
|
|
|
timeoutMs?: number;
|
|
|
|
}
|
|
|
|
) {
|
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
|
|
|
const controller = new AbortController();
|
|
|
|
if (optionsArg.timeoutMs) {
|
|
|
|
plugins.smartdelay.delayFor(optionsArg.timeoutMs).then(() => {
|
|
|
|
controller.abort();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
let cachedResponseDeferred = plugins.smartpromise.defer<Response>();
|
|
|
|
let cacheUsed = false;
|
|
|
|
if (optionsArg.useCache && this.cacheStore.check(urlArg)) {
|
2022-05-29 19:29:42 +00:00
|
|
|
const responseBuffer: ArrayBuffer = await this.cacheStore.get(urlArg);
|
2022-05-29 18:22:42 +00:00
|
|
|
cachedResponseDeferred.resolve(new Response(responseBuffer, {}));
|
|
|
|
} else {
|
|
|
|
cachedResponseDeferred.resolve(null);
|
|
|
|
}
|
2022-05-29 19:29:42 +00:00
|
|
|
let response: Response = await fetch(urlArg, {
|
2022-05-29 18:22:42 +00:00
|
|
|
signal: controller.signal,
|
|
|
|
method: optionsArg.method,
|
|
|
|
headers: {
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
...(optionsArg.headers || {}),
|
|
|
|
},
|
|
|
|
body: optionsArg.body,
|
2022-05-29 19:29:42 +00:00
|
|
|
}).catch(async (err) => {
|
|
|
|
if (optionsArg.useCache && (await cachedResponseDeferred.promise)) {
|
2022-05-29 18:22:42 +00:00
|
|
|
cacheUsed = true;
|
|
|
|
const cachedResponse = cachedResponseDeferred.promise;
|
|
|
|
return cachedResponse;
|
|
|
|
} else {
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
});
|
2022-05-29 19:29:42 +00:00
|
|
|
if (optionsArg.useCache && (await cachedResponseDeferred.promise) && response.status === 500) {
|
|
|
|
cacheUsed = true;
|
|
|
|
response = await cachedResponseDeferred.promise;
|
|
|
|
}
|
2022-05-29 18:22:42 +00:00
|
|
|
if (!cacheUsed && optionsArg.useCache && response.status < 300) {
|
|
|
|
const buffer = await response.clone().arrayBuffer();
|
|
|
|
await this.cacheStore.set(urlArg, buffer);
|
|
|
|
}
|
|
|
|
console.log(`${urlArg} answers with status: ${response.status}`);
|
|
|
|
return response;
|
|
|
|
}
|
|
|
|
|
2018-11-30 16:12:48 +00:00
|
|
|
/**
|
2022-05-29 18:22:42 +00:00
|
|
|
* a multi endpoint, fault tolerant request function
|
2018-11-30 16:12:48 +00:00
|
|
|
*/
|
2022-05-29 18:22:42 +00:00
|
|
|
public async requestMultiEndpoint(
|
2018-11-30 16:12:48 +00:00
|
|
|
urlArg: string | string[],
|
|
|
|
optionsArg: {
|
|
|
|
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
2019-02-15 11:39:29 +00:00
|
|
|
body?: any;
|
2022-02-10 17:02:07 +00:00
|
|
|
headers?: HeadersInit;
|
2018-11-30 16:12:48 +00:00
|
|
|
}
|
2019-06-04 13:40:30 +00:00
|
|
|
): Promise<Response> {
|
2022-03-16 15:23:32 +00:00
|
|
|
await WebRequest.neededPolyfillsLoadedDeferred.promise;
|
2018-11-30 16:12:48 +00:00
|
|
|
let allUrls: string[];
|
|
|
|
let usedUrlIndex = 0;
|
|
|
|
|
|
|
|
// determine what we got
|
|
|
|
if (Array.isArray(urlArg)) {
|
|
|
|
allUrls = urlArg;
|
|
|
|
} else {
|
|
|
|
allUrls = [urlArg];
|
|
|
|
}
|
|
|
|
|
2018-12-04 16:35:40 +00:00
|
|
|
const requestHistory: string[] = []; // keep track of the request history
|
2018-11-30 16:12:48 +00:00
|
|
|
|
2019-06-04 13:40:30 +00:00
|
|
|
const doHistoryCheck = async (
|
|
|
|
// check history for a
|
2018-12-04 16:35:40 +00:00
|
|
|
historyEntryTypeArg: string
|
2018-11-30 16:12:48 +00:00
|
|
|
) => {
|
|
|
|
requestHistory.push(historyEntryTypeArg);
|
2018-12-04 16:35:40 +00:00
|
|
|
if (historyEntryTypeArg === '429') {
|
2019-06-04 13:40:30 +00:00
|
|
|
console.log('got 429, so waiting a little bit.');
|
|
|
|
await plugins.smartdelay.delayFor(Math.floor(Math.random() * (2000 - 1000 + 1)) + 1000); // wait between 1 and 10 seconds
|
2018-12-04 16:35:40 +00:00
|
|
|
}
|
2018-11-30 16:12:48 +00:00
|
|
|
|
|
|
|
let numOfHistoryType = 0;
|
|
|
|
for (const entry of requestHistory) {
|
|
|
|
if (entry === historyEntryTypeArg) numOfHistoryType++;
|
|
|
|
}
|
2019-06-04 13:40:30 +00:00
|
|
|
if (numOfHistoryType > 2 * allUrls.length * usedUrlIndex) {
|
2018-11-30 16:12:48 +00:00
|
|
|
usedUrlIndex++;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// lets go recursive
|
2022-02-10 17:02:07 +00:00
|
|
|
const doRequest = async (urlToUse: string): Promise<any> => {
|
2018-11-30 16:12:48 +00:00
|
|
|
if (!urlToUse) {
|
|
|
|
throw new Error('request failed permanently');
|
|
|
|
}
|
2019-09-27 19:20:06 +00:00
|
|
|
console.log(`Getting ${urlToUse} with method ${optionsArg.method}`);
|
2018-11-30 16:12:48 +00:00
|
|
|
const response = await fetch(urlToUse, {
|
|
|
|
method: optionsArg.method,
|
|
|
|
headers: {
|
2020-10-09 10:14:45 +00:00
|
|
|
'Content-Type': 'application/json',
|
2022-02-10 17:45:22 +00:00
|
|
|
...(optionsArg.headers || {}),
|
2019-02-15 11:39:29 +00:00
|
|
|
},
|
2020-10-09 10:14:45 +00:00
|
|
|
body: optionsArg.body,
|
2018-11-30 16:12:48 +00:00
|
|
|
});
|
2018-12-04 16:35:40 +00:00
|
|
|
console.log(`${urlToUse} answers with status: ${response.status}`);
|
|
|
|
|
2018-12-03 17:35:30 +00:00
|
|
|
if (response.status >= 200 && response.status < 300) {
|
2018-12-04 16:35:40 +00:00
|
|
|
return response;
|
|
|
|
} else {
|
2020-10-15 18:46:21 +00:00
|
|
|
// lets perform a history check to determine failed urls
|
2018-12-04 16:35:40 +00:00
|
|
|
await doHistoryCheck(response.status.toString());
|
2020-10-15 18:46:21 +00:00
|
|
|
// lets fire the request
|
|
|
|
const result = await doRequest(allUrls[usedUrlIndex]);
|
|
|
|
return result;
|
2018-11-30 16:12:48 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2019-09-27 18:16:28 +00:00
|
|
|
const finalResponse: Response = await doRequest(allUrls[usedUrlIndex]);
|
2019-06-04 13:40:30 +00:00
|
|
|
return finalResponse;
|
2018-11-30 16:12:48 +00:00
|
|
|
}
|
|
|
|
}
|