feat: Implement comprehensive web request handling with caching, retry, and interceptors

- Added cache strategies: NetworkFirst, CacheFirst, StaleWhileRevalidate, NetworkOnly, and CacheOnly.
- Introduced InterceptorManager for managing request, response, and error interceptors.
- Developed RetryManager for handling request retries with customizable backoff strategies.
- Implemented RequestDeduplicator to prevent simultaneous identical requests.
- Created timeout utilities for handling request timeouts.
- Enhanced WebrequestClient to support global interceptors, caching, and retry logic.
- Added convenience methods for common HTTP methods (GET, POST, PUT, DELETE) with JSON handling.
- Established a fetch-compatible webrequest function for seamless integration.
- Defined core type structures for caching, retry options, interceptors, and web request configurations.
This commit is contained in:
2025-10-20 09:59:24 +00:00
parent e228ed4ba0
commit 54afcc46e2
30 changed files with 18693 additions and 4031 deletions

172
ts/cache/cache.headers.ts vendored Normal file
View File

@@ -0,0 +1,172 @@
/**
* HTTP Cache Header parsing and utilities
* Implements RFC 7234 (HTTP Caching)
*/
import type { ICacheMetadata } from '../webrequest.types.js';
/**
* Parse Cache-Control header into metadata
*/
export function parseCacheControl(
cacheControlHeader: string | null,
): Partial<ICacheMetadata> {
const metadata: Partial<ICacheMetadata> = {
maxAge: 0,
immutable: false,
noCache: false,
noStore: false,
mustRevalidate: false,
};
if (!cacheControlHeader) {
return metadata;
}
const directives = cacheControlHeader
.toLowerCase()
.split(',')
.map((d) => d.trim());
for (const directive of directives) {
if (directive === 'no-cache') {
metadata.noCache = true;
} else if (directive === 'no-store') {
metadata.noStore = true;
} else if (directive === 'immutable') {
metadata.immutable = true;
} else if (directive === 'must-revalidate') {
metadata.mustRevalidate = true;
} else if (directive.startsWith('max-age=')) {
const maxAge = parseInt(directive.split('=')[1], 10);
if (!isNaN(maxAge)) {
metadata.maxAge = maxAge * 1000; // Convert to milliseconds
}
}
}
return metadata;
}
/**
* Parse Expires header into timestamp
*/
export function parseExpires(expiresHeader: string | null): number | undefined {
if (!expiresHeader) {
return undefined;
}
try {
const date = new Date(expiresHeader);
return date.getTime();
} catch {
return undefined;
}
}
/**
* Extract cache metadata from response headers
*/
export function extractCacheMetadata(headers: Headers): ICacheMetadata {
const cacheControl = headers.get('cache-control');
const expires = headers.get('expires');
const etag = headers.get('etag');
const lastModified = headers.get('last-modified');
const metadata = parseCacheControl(cacheControl);
// If no max-age from Cache-Control, try Expires header
if (metadata.maxAge === 0 && expires) {
const expiresTime = parseExpires(expires);
if (expiresTime) {
metadata.maxAge = Math.max(0, expiresTime - Date.now());
}
}
return {
maxAge: metadata.maxAge || 0,
etag: etag || undefined,
lastModified: lastModified || undefined,
immutable: metadata.immutable || false,
noCache: metadata.noCache || false,
noStore: metadata.noStore || false,
mustRevalidate: metadata.mustRevalidate || false,
};
}
/**
* Check if a cached response is still fresh
*/
export function isFresh(
cacheEntry: { timestamp: number; maxAge?: number },
metadata: ICacheMetadata,
): boolean {
// no-store means never cache
if (metadata.noStore) {
return false;
}
// If immutable, it's always fresh
if (metadata.immutable) {
return true;
}
const age = Date.now() - cacheEntry.timestamp;
const maxAge = cacheEntry.maxAge || metadata.maxAge || 0;
// If no max-age specified, consider stale
if (maxAge === 0) {
return false;
}
return age < maxAge;
}
/**
* Check if revalidation is required
*/
export function requiresRevalidation(metadata: ICacheMetadata): boolean {
return metadata.noCache || metadata.mustRevalidate;
}
/**
* Create conditional request headers for revalidation
*/
export function createConditionalHeaders(cacheEntry: {
etag?: string;
lastModified?: string;
}): HeadersInit {
const headers: Record<string, string> = {};
if (cacheEntry.etag) {
headers['if-none-match'] = cacheEntry.etag;
}
if (cacheEntry.lastModified) {
headers['if-modified-since'] = cacheEntry.lastModified;
}
return headers;
}
/**
* Convert Headers object to plain object for storage
*/
export function headersToObject(headers: Headers): Record<string, string> {
const obj: Record<string, string> = {};
headers.forEach((value, key) => {
obj[key] = value;
});
return obj;
}
/**
* Convert plain object back to Headers
*/
export function objectToHeaders(obj: Record<string, string>): Headers {
const headers = new Headers();
Object.entries(obj).forEach(([key, value]) => {
headers.set(key, value);
});
return headers;
}

156
ts/cache/cache.manager.ts vendored Normal file
View File

@@ -0,0 +1,156 @@
/**
* Cache manager - orchestrates caching logic
*/
import type {
ICacheOptions,
TCacheStrategy,
TStandardCacheMode,
} from '../webrequest.types.js';
import { CacheStore } from './cache.store.js';
import {
getStrategyHandler,
type IStrategyContext,
type IStrategyResult,
} from './cache.strategies.js';
import { extractCacheMetadata } from './cache.headers.js';
export class CacheManager {
private cacheStore: CacheStore;
constructor(dbName?: string, storeName?: string) {
this.cacheStore = new CacheStore(dbName, storeName);
}
/**
* Execute a request with caching
*/
public async execute(
request: Request,
options: ICacheOptions & { logging?: boolean },
fetchFn: (request: Request) => Promise<Response>,
): Promise<IStrategyResult> {
// Determine the cache strategy
const strategy = this.determineStrategy(request, options);
// If no caching (no-store or network-only), bypass cache
if (strategy === 'network-only') {
const response = await fetchFn(request);
return {
response,
fromCache: false,
revalidated: false,
};
}
// Generate cache key
const cacheKey = this.generateCacheKey(request, options);
// Get strategy handler
const handler = getStrategyHandler(strategy);
// Execute strategy
const context: IStrategyContext = {
request,
cacheKey,
cacheStore: this.cacheStore,
fetchFn,
logging: options.logging,
};
return await handler.execute(context);
}
/**
* Determine the caching strategy based on options and request
*/
private determineStrategy(
request: Request,
options: ICacheOptions,
): TCacheStrategy {
// If explicit strategy provided, use it
if (options.cacheStrategy) {
return options.cacheStrategy;
}
// Map standard cache modes to strategies
if (options.cache) {
return this.mapCacheModeToStrategy(options.cache);
}
// Check request cache mode
if (request.cache) {
return this.mapCacheModeToStrategy(request.cache as TStandardCacheMode);
}
// Default strategy
return 'network-first';
}
/**
* Map standard fetch cache modes to our strategies
*/
private mapCacheModeToStrategy(
cacheMode: TStandardCacheMode,
): TCacheStrategy {
switch (cacheMode) {
case 'default':
return 'network-first';
case 'no-store':
case 'reload':
return 'network-only';
case 'no-cache':
return 'network-first'; // Will use revalidation
case 'force-cache':
return 'cache-first';
case 'only-if-cached':
return 'cache-only';
default:
return 'network-first';
}
}
/**
* Generate cache key
*/
private generateCacheKey(request: Request, options: ICacheOptions): string {
// If custom cache key provided
if (options.cacheKey) {
if (typeof options.cacheKey === 'function') {
return options.cacheKey(request);
}
return options.cacheKey;
}
// Default cache key generation
return this.cacheStore.generateCacheKey(request);
}
/**
* Clear the cache
*/
public async clear(): Promise<void> {
await this.cacheStore.clear();
}
/**
* Delete a specific cache entry
*/
public async delete(cacheKey: string): Promise<void> {
await this.cacheStore.delete(cacheKey);
}
/**
* Check if a cache entry exists
*/
public async has(cacheKey: string): Promise<boolean> {
return await this.cacheStore.has(cacheKey);
}
/**
* Get the underlying cache store
*/
public getStore(): CacheStore {
return this.cacheStore;
}
}

154
ts/cache/cache.store.ts vendored Normal file
View File

@@ -0,0 +1,154 @@
/**
* Cache storage layer using IndexedDB via @push.rocks/webstore
*/
import * as plugins from '../webrequest.plugins.js';
import type { ICacheEntry } from '../webrequest.types.js';
export class CacheStore {
private webstore: plugins.webstore.WebStore;
private initPromise: Promise<void>;
constructor(dbName: string = 'webrequest-v4', storeName: string = 'cache') {
this.webstore = new plugins.webstore.WebStore({
dbName,
storeName,
});
// Initialize the store
this.initPromise = this.init();
}
/**
* Initialize the store
*/
private async init(): Promise<void> {
// WebStore handles initialization internally
// This method exists for future extension if needed
}
/**
* Generate a cache key from a request
*/
public generateCacheKey(request: Request): string {
// Use URL + method as the base key
const url = request.url;
const method = request.method;
// For GET requests, just use the URL
if (method === 'GET') {
return url;
}
// For other methods, include the method
return `${method}:${url}`;
}
/**
* Store a response in the cache
*/
public async set(cacheKey: string, entry: ICacheEntry): Promise<void> {
await this.initPromise;
await this.webstore.set(cacheKey, entry);
}
/**
* Retrieve a cached response
*/
public async get(cacheKey: string): Promise<ICacheEntry | null> {
await this.initPromise;
try {
const entry = (await this.webstore.get(cacheKey)) as ICacheEntry;
return entry || null;
} catch (error) {
// If entry doesn't exist or is corrupted, return null
return null;
}
}
/**
* Check if a cache entry exists
*/
public async has(cacheKey: string): Promise<boolean> {
await this.initPromise;
return await this.webstore.check(cacheKey);
}
/**
* Delete a cache entry
*/
public async delete(cacheKey: string): Promise<void> {
await this.initPromise;
await this.webstore.delete(cacheKey);
}
/**
* Clear all cache entries
*/
public async clear(): Promise<void> {
await this.initPromise;
await this.webstore.clear();
}
/**
* Create a Response object from a cache entry
*/
public responseFromCacheEntry(entry: ICacheEntry): Response {
const headers = new Headers(entry.headers);
return new Response(entry.response, {
status: entry.status,
statusText: entry.statusText,
headers,
});
}
/**
* Create a cache entry from a Response object
*/
public async cacheEntryFromResponse(
url: string,
response: Response,
metadata?: { maxAge?: number; etag?: string; lastModified?: string },
): Promise<ICacheEntry> {
// Clone the response so we can read it multiple times
const clonedResponse = response.clone();
const buffer = await clonedResponse.arrayBuffer();
// Extract headers
const headers: Record<string, string> = {};
clonedResponse.headers.forEach((value, key) => {
headers[key] = value;
});
return {
response: buffer,
headers,
timestamp: Date.now(),
etag: metadata?.etag || clonedResponse.headers.get('etag') || undefined,
lastModified:
metadata?.lastModified ||
clonedResponse.headers.get('last-modified') ||
undefined,
maxAge: metadata?.maxAge,
url,
status: clonedResponse.status,
statusText: clonedResponse.statusText,
};
}
/**
* Prune expired entries (garbage collection)
* Returns the number of entries deleted
*/
public async pruneExpired(): Promise<number> {
await this.initPromise;
// Note: WebStore doesn't provide a way to list all keys
// This would need to be implemented if we want automatic cleanup
// For now, we rely on individual entry checks
return 0;
}
}

377
ts/cache/cache.strategies.ts vendored Normal file
View File

@@ -0,0 +1,377 @@
/**
* Cache strategy implementations
*/
import type {
ICacheEntry,
ICacheMetadata,
TCacheStrategy,
} from '../webrequest.types.js';
import { CacheStore } from './cache.store.js';
import {
extractCacheMetadata,
isFresh,
requiresRevalidation,
createConditionalHeaders,
headersToObject,
} from './cache.headers.js';
export interface IStrategyContext {
request: Request;
cacheKey: string;
cacheStore: CacheStore;
fetchFn: (request: Request) => Promise<Response>;
logging?: boolean;
}
export interface IStrategyResult {
response: Response;
fromCache: boolean;
revalidated: boolean;
}
/**
* Base strategy handler interface
*/
export interface ICacheStrategyHandler {
execute(context: IStrategyContext): Promise<IStrategyResult>;
}
/**
* Network-First Strategy
* Try network first, fallback to cache on failure
*/
export class NetworkFirstStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
try {
// Try network first
const response = await context.fetchFn(context.request);
// If successful, cache it
if (response.ok) {
await this.cacheResponse(context, response);
}
return {
response,
fromCache: false,
revalidated: false,
};
} catch (error) {
// Network failed, try cache
if (context.logging) {
console.log('[webrequest] Network failed, trying cache:', error);
}
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
// No cache available, re-throw error
throw error;
}
}
private async cacheResponse(
context: IStrategyContext,
response: Response,
): Promise<void> {
const metadata = extractCacheMetadata(response.headers);
// Don't cache if no-store
if (metadata.noStore) {
return;
}
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
}
/**
* Cache-First Strategy
* Check cache first, fetch if miss or stale
*/
export class CacheFirstStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
// Check cache first
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
const metadata = extractCacheMetadata(new Headers(cachedEntry.headers));
// Check if cache is fresh
if (isFresh(cachedEntry, metadata)) {
if (context.logging) {
console.log('[webrequest] Cache hit (fresh):', context.request.url);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
// If requires revalidation, check with server
if (
requiresRevalidation(metadata) &&
(cachedEntry.etag || cachedEntry.lastModified)
) {
return await this.revalidate(context, cachedEntry);
}
}
// Cache miss or stale, fetch from network
if (context.logging) {
console.log('[webrequest] Cache miss, fetching:', context.request.url);
}
const response = await context.fetchFn(context.request);
// Cache the response
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
return {
response,
fromCache: false,
revalidated: false,
};
}
private async revalidate(
context: IStrategyContext,
cachedEntry: ICacheEntry,
): Promise<IStrategyResult> {
const conditionalHeaders = createConditionalHeaders(cachedEntry);
// Create a new request with conditional headers
const revalidateRequest = new Request(context.request.url, {
method: context.request.method,
headers: {
...headersToObject(context.request.headers),
...conditionalHeaders,
},
});
try {
const response = await context.fetchFn(revalidateRequest);
// 304 Not Modified - cache is still valid
if (response.status === 304) {
if (context.logging) {
console.log(
'[webrequest] Cache revalidated (304):',
context.request.url,
);
}
// Update timestamp
cachedEntry.timestamp = Date.now();
await context.cacheStore.set(context.cacheKey, cachedEntry);
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: true,
};
}
// Response changed, cache the new one
if (response.ok) {
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
}
return {
response,
fromCache: false,
revalidated: true,
};
} catch (error) {
// Revalidation failed, use cached response
if (context.logging) {
console.log('[webrequest] Revalidation failed, using cache:', error);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
}
}
/**
* Stale-While-Revalidate Strategy
* Return cache immediately, update in background
*/
export class StaleWhileRevalidateStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
// Return cached response immediately
const cachedResponse =
context.cacheStore.responseFromCacheEntry(cachedEntry);
// Revalidate in background
this.revalidateInBackground(context, cachedEntry).catch((error) => {
if (context.logging) {
console.warn('[webrequest] Background revalidation failed:', error);
}
});
return {
response: cachedResponse,
fromCache: true,
revalidated: false,
};
}
// No cache, fetch from network
const response = await context.fetchFn(context.request);
// Cache the response
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore && response.ok) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
return {
response,
fromCache: false,
revalidated: false,
};
}
private async revalidateInBackground(
context: IStrategyContext,
cachedEntry: ICacheEntry,
): Promise<void> {
const metadata = extractCacheMetadata(new Headers(cachedEntry.headers));
// Check if revalidation is needed
if (isFresh(cachedEntry, metadata) && !requiresRevalidation(metadata)) {
return;
}
try {
const response = await context.fetchFn(context.request);
if (response.ok) {
const newMetadata = extractCacheMetadata(response.headers);
if (!newMetadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
newMetadata,
);
await context.cacheStore.set(context.cacheKey, entry);
if (context.logging) {
console.log(
'[webrequest] Background revalidation complete:',
context.request.url,
);
}
}
}
} catch (error) {
// Background revalidation failed, keep existing cache
if (context.logging) {
console.warn('[webrequest] Background revalidation failed:', error);
}
}
}
}
/**
* Network-Only Strategy
* Never use cache
*/
export class NetworkOnlyStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const response = await context.fetchFn(context.request);
return {
response,
fromCache: false,
revalidated: false,
};
}
}
/**
* Cache-Only Strategy
* Only use cache, fail if miss
*/
export class CacheOnlyStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (!cachedEntry) {
throw new Error(
`Cache miss for ${context.request.url} (cache-only mode)`,
);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
}
/**
* Get strategy handler for a given strategy type
*/
export function getStrategyHandler(
strategy: TCacheStrategy,
): ICacheStrategyHandler {
switch (strategy) {
case 'network-first':
return new NetworkFirstStrategy();
case 'cache-first':
return new CacheFirstStrategy();
case 'stale-while-revalidate':
return new StaleWhileRevalidateStrategy();
case 'network-only':
return new NetworkOnlyStrategy();
case 'cache-only':
return new CacheOnlyStrategy();
default:
return new NetworkFirstStrategy();
}
}