feat: Implement comprehensive web request handling with caching, retry, and interceptors
- Added cache strategies: NetworkFirst, CacheFirst, StaleWhileRevalidate, NetworkOnly, and CacheOnly. - Introduced InterceptorManager for managing request, response, and error interceptors. - Developed RetryManager for handling request retries with customizable backoff strategies. - Implemented RequestDeduplicator to prevent simultaneous identical requests. - Created timeout utilities for handling request timeouts. - Enhanced WebrequestClient to support global interceptors, caching, and retry logic. - Added convenience methods for common HTTP methods (GET, POST, PUT, DELETE) with JSON handling. - Established a fetch-compatible webrequest function for seamless integration. - Defined core type structures for caching, retry options, interceptors, and web request configurations.
This commit is contained in:
253
ts/index.ts
253
ts/index.ts
@@ -1,220 +1,47 @@
|
||||
import * as plugins from './webrequest.plugins.js';
|
||||
|
||||
export interface IWebrequestContructorOptions {
|
||||
logging?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* web request
|
||||
* @push.rocks/webrequest v4
|
||||
* Modern, fetch-compatible web request library with intelligent caching
|
||||
*/
|
||||
export class WebRequest {
|
||||
|
||||
public cacheStore = new plugins.webstore.WebStore({
|
||||
dbName: 'webrequest',
|
||||
storeName: 'webrequest',
|
||||
});
|
||||
// Main exports
|
||||
export { webrequest } from './webrequest.function.js';
|
||||
export { WebrequestClient } from './webrequest.client.js';
|
||||
|
||||
public options: IWebrequestContructorOptions;
|
||||
// Type exports
|
||||
export type {
|
||||
IWebrequestOptions,
|
||||
ICacheOptions,
|
||||
IRetryOptions,
|
||||
IInterceptors,
|
||||
TCacheStrategy,
|
||||
TStandardCacheMode,
|
||||
TBackoffStrategy,
|
||||
TWebrequestResult,
|
||||
IWebrequestSuccess,
|
||||
IWebrequestError,
|
||||
ICacheEntry,
|
||||
ICacheMetadata,
|
||||
} from './webrequest.types.js';
|
||||
|
||||
constructor(public optionsArg: IWebrequestContructorOptions = {}) {
|
||||
this.options = {
|
||||
logging: true,
|
||||
...optionsArg,
|
||||
};
|
||||
}
|
||||
export type {
|
||||
TRequestInterceptor,
|
||||
TResponseInterceptor,
|
||||
TErrorInterceptor,
|
||||
} from './interceptors/interceptor.types.js';
|
||||
|
||||
public async getJson(urlArg: string, useCacheArg: boolean = false) {
|
||||
const response: Response = await this.request(urlArg, {
|
||||
method: 'GET',
|
||||
useCache: useCacheArg,
|
||||
});
|
||||
const responseText = await response.text();
|
||||
const responseResult = plugins.smartjson.parse(responseText);
|
||||
return responseResult;
|
||||
}
|
||||
// Advanced exports for custom implementations
|
||||
export { CacheManager } from './cache/cache.manager.js';
|
||||
export { CacheStore } from './cache/cache.store.js';
|
||||
export { RetryManager } from './retry/retry.manager.js';
|
||||
export { InterceptorManager } from './interceptors/interceptor.manager.js';
|
||||
export { RequestDeduplicator } from './utils/deduplicator.js';
|
||||
|
||||
/**
|
||||
* postJson
|
||||
*/
|
||||
public async postJson(urlArg: string, requestBody?: any, useCacheArg: boolean = false) {
|
||||
const response: Response = await this.request(urlArg, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: plugins.smartjson.stringify(requestBody),
|
||||
useCache: useCacheArg,
|
||||
});
|
||||
const responseText = await response.text();
|
||||
const responseResult = plugins.smartjson.parse(responseText);
|
||||
return responseResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* put js
|
||||
*/
|
||||
public async putJson(urlArg: string, requestBody?: any, useStoreAsFallback: boolean = false) {
|
||||
const response: Response = await this.request(urlArg, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: plugins.smartjson.stringify(requestBody),
|
||||
});
|
||||
const responseText = await response.text();
|
||||
const responseResult = plugins.smartjson.parse(responseText);
|
||||
return responseResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* put js
|
||||
*/
|
||||
public async deleteJson(urlArg: string, useStoreAsFallback: boolean = false) {
|
||||
const response: Response = await this.request(urlArg, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
method: 'GET',
|
||||
});
|
||||
const responseText = await response.text();
|
||||
const responseResult = plugins.smartjson.parse(responseText);
|
||||
return responseResult;
|
||||
}
|
||||
|
||||
public async request(
|
||||
urlArg: string,
|
||||
optionsArg: {
|
||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
||||
body?: any;
|
||||
headers?: HeadersInit;
|
||||
useCache?: boolean;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
) {
|
||||
optionsArg = {
|
||||
timeoutMs: 60000,
|
||||
useCache: false,
|
||||
...optionsArg,
|
||||
};
|
||||
|
||||
let controller = new AbortController();
|
||||
plugins.smartdelay.delayFor(optionsArg.timeoutMs).then(() => {
|
||||
controller.abort();
|
||||
});
|
||||
let cachedResponseDeferred = plugins.smartpromise.defer<Response>();
|
||||
let cacheUsed = false;
|
||||
if (optionsArg.useCache && (await this.cacheStore.check(urlArg))) {
|
||||
const responseBuffer: ArrayBuffer = await this.cacheStore.get(urlArg);
|
||||
cachedResponseDeferred.resolve(new Response(responseBuffer, {}));
|
||||
} else {
|
||||
cachedResponseDeferred.resolve(null);
|
||||
}
|
||||
let response: Response = await fetch(urlArg, {
|
||||
signal: controller.signal,
|
||||
method: optionsArg.method,
|
||||
headers: {
|
||||
...(optionsArg.headers || {}),
|
||||
},
|
||||
body: optionsArg.body,
|
||||
})
|
||||
.catch(async (err) => {
|
||||
if (optionsArg.useCache && (await cachedResponseDeferred.promise)) {
|
||||
cacheUsed = true;
|
||||
const cachedResponse = cachedResponseDeferred.promise;
|
||||
return cachedResponse;
|
||||
} else {
|
||||
return err;
|
||||
}
|
||||
});
|
||||
if (optionsArg.useCache && (await cachedResponseDeferred.promise) && response.status === 500) {
|
||||
cacheUsed = true;
|
||||
response = await cachedResponseDeferred.promise;
|
||||
}
|
||||
if (!cacheUsed && optionsArg.useCache && response.status < 300) {
|
||||
const buffer = await response.clone().arrayBuffer();
|
||||
await this.cacheStore.set(urlArg, buffer);
|
||||
}
|
||||
this.log(`${urlArg} answers with status: ${response.status}`);
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* a multi endpoint, fault tolerant request function
|
||||
*/
|
||||
public async requestMultiEndpoint(
|
||||
urlArg: string | string[],
|
||||
optionsArg: {
|
||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
||||
body?: any;
|
||||
headers?: HeadersInit;
|
||||
}
|
||||
): Promise<Response> {
|
||||
|
||||
let allUrls: string[];
|
||||
let usedUrlIndex = 0;
|
||||
|
||||
// determine what we got
|
||||
if (Array.isArray(urlArg)) {
|
||||
allUrls = urlArg;
|
||||
} else {
|
||||
allUrls = [urlArg];
|
||||
}
|
||||
|
||||
const requestHistory: string[] = []; // keep track of the request history
|
||||
|
||||
const doHistoryCheck = async (
|
||||
// check history for a
|
||||
historyEntryTypeArg: string
|
||||
) => {
|
||||
requestHistory.push(historyEntryTypeArg);
|
||||
if (historyEntryTypeArg === '429') {
|
||||
console.log('got 429, so waiting a little bit.');
|
||||
await plugins.smartdelay.delayFor(Math.floor(Math.random() * (2000 - 1000 + 1)) + 1000); // wait between 1 and 10 seconds
|
||||
}
|
||||
|
||||
let numOfHistoryType = 0;
|
||||
for (const entry of requestHistory) {
|
||||
if (entry === historyEntryTypeArg) numOfHistoryType++;
|
||||
}
|
||||
if (numOfHistoryType > 2 * allUrls.length * usedUrlIndex) {
|
||||
usedUrlIndex++;
|
||||
}
|
||||
};
|
||||
|
||||
// lets go recursive
|
||||
const doRequest = async (urlToUse: string): Promise<any> => {
|
||||
if (!urlToUse) {
|
||||
throw new Error('request failed permanently');
|
||||
}
|
||||
this.log(`Getting ${urlToUse} with method ${optionsArg.method}`);
|
||||
const response = await fetch(urlToUse, {
|
||||
method: optionsArg.method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(optionsArg.headers || {}),
|
||||
},
|
||||
body: optionsArg.body,
|
||||
});
|
||||
this.log(`${urlToUse} answers with status: ${response.status}`);
|
||||
|
||||
if (response.status >= 200 && response.status < 300) {
|
||||
return response;
|
||||
} else {
|
||||
// lets perform a history check to determine failed urls
|
||||
await doHistoryCheck(response.status.toString());
|
||||
// lets fire the request
|
||||
const result = await doRequest(allUrls[usedUrlIndex]);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
const finalResponse: Response = await doRequest(allUrls[usedUrlIndex]);
|
||||
return finalResponse;
|
||||
}
|
||||
|
||||
public log(logArg: string) {
|
||||
if (this.options.logging) {
|
||||
console.log(logArg);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Cache utilities
|
||||
export {
|
||||
extractCacheMetadata,
|
||||
isFresh,
|
||||
requiresRevalidation,
|
||||
createConditionalHeaders,
|
||||
headersToObject,
|
||||
objectToHeaders,
|
||||
} from './cache/cache.headers.js';
|
||||
|
||||
Reference in New Issue
Block a user