feat: Implement comprehensive web request handling with caching, retry, and interceptors

- Added cache strategies: NetworkFirst, CacheFirst, StaleWhileRevalidate, NetworkOnly, and CacheOnly.
- Introduced InterceptorManager for managing request, response, and error interceptors.
- Developed RetryManager for handling request retries with customizable backoff strategies.
- Implemented RequestDeduplicator to prevent simultaneous identical requests.
- Created timeout utilities for handling request timeouts.
- Enhanced WebrequestClient to support global interceptors, caching, and retry logic.
- Added convenience methods for common HTTP methods (GET, POST, PUT, DELETE) with JSON handling.
- Established a fetch-compatible webrequest function for seamless integration.
- Defined core type structures for caching, retry options, interceptors, and web request configurations.
This commit is contained in:
2025-10-20 09:59:24 +00:00
parent e228ed4ba0
commit 54afcc46e2
30 changed files with 18693 additions and 4031 deletions

View File

@@ -6,8 +6,8 @@ on:
- '**'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
@@ -26,7 +26,7 @@ jobs:
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
pnpm install -g @ship.zone/npmci
- name: Run npm prepare
run: npmci npm prepare

View File

@@ -6,8 +6,8 @@ on:
- '*'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
@@ -26,7 +26,7 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Audit production dependencies
@@ -54,7 +54,7 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Test stable
@@ -82,7 +82,7 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Release
@@ -104,7 +104,7 @@ jobs:
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Code quality
@@ -119,6 +119,6 @@ jobs:
run: |
npmci node install stable
npmci npm install
pnpm install -g @gitzone/tsdoc
pnpm install -g @git.zone/tsdoc
npmci command tsdoc
continue-on-error: true

7
.gitignore vendored
View File

@@ -3,7 +3,6 @@
# artifacts
coverage/
public/
pages/
# installs
node_modules/
@@ -17,4 +16,8 @@ node_modules/
dist/
dist_*/
# custom
# AI
.claude/
.serena/
#------# custom

7242
deno.lock generated Normal file

File diff suppressed because it is too large Load Diff

339
migration-v4.md Normal file
View File

@@ -0,0 +1,339 @@
# Migration Guide: v3 → v4
## Overview
Version 4.0 is a complete modernization of `@push.rocks/webrequest`, bringing it in line with modern web standards while maintaining backward compatibility through a deprecated API layer.
## What's New in v4
### Core Improvements
- **Fetch-Compatible API**: Drop-in replacement for native `fetch()` with enhanced features
- **Intelligent HTTP Caching**: Respects `Cache-Control`, `ETag`, `Last-Modified`, and `Expires` headers
- **Multiple Cache Strategies**: network-first, cache-first, stale-while-revalidate, network-only, cache-only
- **Advanced Retry System**: Configurable retry with exponential/linear/constant backoff
- **Request/Response Interceptors**: Middleware pattern for transforming requests and responses
- **Request Deduplication**: Automatically deduplicate simultaneous identical requests
- **TypeScript Generics**: Type-safe response parsing with `webrequest.getJson<T>()`
- **Better Fault Tolerance**: Enhanced multi-endpoint fallback with retry strategies
### Bug Fixes
- Fixed: `deleteJson()` now correctly uses `DELETE` method instead of `GET`
## Migration Path
### Option 1: Quick Migration (Recommended)
The v3 `WebRequest` class is still available but marked as deprecated. Your existing code will continue to work:
```typescript
// This still works in v4 (with deprecation warnings)
import { WebRequest } from '@push.rocks/webrequest';
const client = new WebRequest({ logging: true });
const data = await client.getJson('https://api.example.com/data', true);
```
### Option 2: Migrate to New API
#### Basic Fetch-Compatible Usage
```typescript
// v3
import { WebRequest } from '@push.rocks/webrequest';
const client = new WebRequest();
const response = await client.request('https://api.example.com/data', {
method: 'GET'
});
const data = await response.json();
// v4 - Fetch-compatible
import { webrequest } from '@push.rocks/webrequest';
const response = await webrequest('https://api.example.com/data');
const data = await response.json();
```
#### JSON Convenience Methods
```typescript
// v3
const client = new WebRequest();
const data = await client.getJson('https://api.example.com/data', true);
// v4 - Function API
import { webrequest } from '@push.rocks/webrequest';
const data = await webrequest.getJson('https://api.example.com/data', {
cacheStrategy: 'cache-first'
});
// v4 - Client API (similar to v3)
import { WebrequestClient } from '@push.rocks/webrequest';
const client = new WebrequestClient({ logging: true });
const data = await client.getJson('https://api.example.com/data', {
cacheStrategy: 'cache-first'
});
```
## Feature Comparison
### Caching
```typescript
// v3 - Boolean flag
const data = await client.getJson(url, true); // useCache = true
// v4 - Explicit strategies
const data = await webrequest.getJson(url, {
cacheStrategy: 'cache-first',
cacheMaxAge: 60000 // 60 seconds
});
// v4 - HTTP header-based caching (automatic)
const data = await webrequest.getJson(url, {
cacheStrategy: 'network-first' // Respects Cache-Control headers
});
// v4 - Stale-while-revalidate
const data = await webrequest.getJson(url, {
cacheStrategy: 'stale-while-revalidate' // Return cache, update in background
});
```
### Multi-Endpoint Fallback
```typescript
// v3
const client = new WebRequest();
const response = await client.requestMultiEndpoint(
['https://api1.example.com/data', 'https://api2.example.com/data'],
{ method: 'GET' }
);
// v4
import { webrequest } from '@push.rocks/webrequest';
const response = await webrequest('https://api1.example.com/data', {
fallbackUrls: ['https://api2.example.com/data'],
retry: {
maxAttempts: 3,
backoff: 'exponential'
}
});
```
### Timeout
```typescript
// v3
const response = await client.request(url, {
method: 'GET',
timeoutMs: 30000
});
// v4
const response = await webrequest(url, {
timeout: 30000 // milliseconds
});
```
## New Features in v4
### Retry Strategies
```typescript
import { webrequest } from '@push.rocks/webrequest';
const response = await webrequest('https://api.example.com/data', {
retry: {
maxAttempts: 3,
backoff: 'exponential', // or 'linear', 'constant'
initialDelay: 1000,
maxDelay: 30000,
retryOn: [408, 429, 500, 502, 503, 504], // Status codes to retry
onRetry: (attempt, error, nextDelay) => {
console.log(`Retry attempt ${attempt}, waiting ${nextDelay}ms`);
}
}
});
```
### Request/Response Interceptors
```typescript
import { webrequest } from '@push.rocks/webrequest';
// Add global request interceptor
webrequest.addRequestInterceptor((request) => {
// Add auth header
const headers = new Headers(request.headers);
headers.set('Authorization', `Bearer ${getToken()}`);
return new Request(request, { headers });
});
// Add global response interceptor
webrequest.addResponseInterceptor((response) => {
console.log(`Response: ${response.status} ${response.url}`);
return response;
});
// Per-request interceptors
const response = await webrequest('https://api.example.com/data', {
interceptors: {
request: [(req) => {
console.log('Sending request:', req.url);
return req;
}],
response: [(res) => {
console.log('Received response:', res.status);
return res;
}]
}
});
```
### Request Deduplication
```typescript
import { webrequest } from '@push.rocks/webrequest';
// Multiple simultaneous identical requests will be deduplicated
const [res1, res2, res3] = await Promise.all([
webrequest('https://api.example.com/data', { deduplicate: true }),
webrequest('https://api.example.com/data', { deduplicate: true }),
webrequest('https://api.example.com/data', { deduplicate: true }),
]);
// Only one actual network request is made
```
### TypeScript Generics
```typescript
import { webrequest } from '@push.rocks/webrequest';
interface User {
id: number;
name: string;
email: string;
}
// Type-safe response parsing
const user = await webrequest.getJson<User>('https://api.example.com/user/1');
// user is typed as User
const users = await webrequest.getJson<User[]>('https://api.example.com/users');
// users is typed as User[]
```
### Custom Cache Keys
```typescript
import { webrequest } from '@push.rocks/webrequest';
const response = await webrequest('https://api.example.com/search?q=test', {
cacheStrategy: 'cache-first',
cacheKey: (request) => {
// Custom cache key based on URL without query params
const url = new URL(request.url);
return `search:${url.searchParams.get('q')}`;
}
});
```
### Advanced Client Configuration
```typescript
import { WebrequestClient } from '@push.rocks/webrequest';
// Create a client with default options
const apiClient = new WebrequestClient({
logging: true,
timeout: 30000,
cacheStrategy: 'network-first',
retry: {
maxAttempts: 3,
backoff: 'exponential'
}
});
// Add global interceptors
apiClient.addRequestInterceptor((request) => {
const headers = new Headers(request.headers);
headers.set('X-API-Key', process.env.API_KEY);
return new Request(request, { headers });
});
// All requests through this client use the configured defaults
const data = await apiClient.getJson('https://api.example.com/data');
```
## Breaking Changes
### 1. Cache API Changes
**Before (v3):**
```typescript
await client.getJson(url, true); // Boolean for cache
```
**After (v4):**
```typescript
await webrequest.getJson(url, {
cacheStrategy: 'cache-first' // Explicit strategy
});
```
### 2. Multi-Endpoint API
**Before (v3):**
```typescript
await client.requestMultiEndpoint([url1, url2], options);
```
**After (v4):**
```typescript
await webrequest(url1, {
fallbackUrls: [url2],
retry: true
});
```
### 3. Constructor Options
**Before (v3):**
```typescript
const client = new WebRequest({ logging: true });
```
**After (v4):**
```typescript
// Function API (no constructor)
import { webrequest } from '@push.rocks/webrequest';
// Or client API
import { WebrequestClient } from '@push.rocks/webrequest';
const client = new WebrequestClient({ logging: true });
```
## Deprecation Timeline
- **v4.0**: `WebRequest` class marked as deprecated but fully functional
- **v4.x**: Continued support with deprecation warnings
- **v5.0**: `WebRequest` class will be removed
## Recommendation
We strongly recommend migrating to the new API to take advantage of:
- Standards-aligned fetch-compatible interface
- Intelligent HTTP caching with header support
- Advanced retry and fault tolerance
- Request/response interceptors
- Better TypeScript support
- Request deduplication
The migration is straightforward, and the new API is more powerful and flexible while being simpler to use.
## Need Help?
- Check the updated [README.md](./readme.md) for comprehensive examples
- Report issues at: https://code.foss.global/push.rocks/webrequest/issues

View File

@@ -30,4 +30,4 @@
"tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
}
}
}

View File

@@ -1,31 +1,29 @@
{
"name": "@push.rocks/webrequest",
"version": "3.0.37",
"version": "4.0.0",
"private": false,
"description": "A module for making secure web requests from browsers with support for caching and fault tolerance.",
"description": "Modern, fetch-compatible web request library with intelligent HTTP caching, retry strategies, and fault tolerance.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
"author": "Lossless GmbH",
"license": "MIT",
"scripts": {
"test": "(tstest test/ --web)",
"test": "(tstest test/ --verbose)",
"build": "(tsbuild --web --allowimplicitany && tsbundle npm)",
"buildDocs": "tsdoc"
},
"devDependencies": {
"@api.global/typedserver": "^3.0.27",
"@git.zone/tsbuild": "^2.1.72",
"@git.zone/tsbuild": "^2.6.8",
"@git.zone/tsbundle": "^2.0.15",
"@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.88",
"@push.rocks/tapbundle": "^5.0.23",
"@git.zone/tstest": "^2.6.2",
"@types/node": "^20.12.7"
},
"dependencies": {
"@push.rocks/smartdelay": "^3.0.5",
"@push.rocks/smartenv": "^5.0.12",
"@push.rocks/smartjson": "^5.0.19",
"@push.rocks/smartjson": "^5.2.0",
"@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/webstore": "^2.0.13"
},
@@ -57,9 +55,16 @@
"multi-endpoint",
"fetch API"
],
"homepage": "https://code.foss.global/push.rocks/webrequest",
"homepage": "https://code.foss.global/push.rocks/webrequest#readme",
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/webrequest.git"
},
"packageManager": "pnpm@10.11.0+sha512.6540583f41cc5f628eb3d9773ecee802f4f9ef9923cc45b69890fb47991d4b092964694ec3a4f738a420c918a333062c8b925d312f42e4f0c263eb603551f977",
"bugs": {
"url": "https://code.foss.global/push.rocks/webrequest/issues"
},
"pnpm": {
"overrides": {}
}
}
}

12256
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

4
pnpm-workspace.yaml Normal file
View File

@@ -0,0 +1,4 @@
onlyBuiltDependencies:
- esbuild
- mongodb-memory-server
- puppeteer

View File

@@ -1,4 +1,5 @@
# @push.rocks/webrequest
securely request from browsers
## Install
@@ -29,7 +30,7 @@ Create an instance of `WebRequest`. You can optionally pass configuration option
```typescript
const webRequest = new WebRequest({
logging: true // Optional: enables logging, defaults to true
logging: true, // Optional: enables logging, defaults to true
});
```
@@ -104,11 +105,11 @@ fetchDataWithCache();
async function requestFromMultipleEndpoints() {
const endpoints = [
'https://api.primary-example.com/data',
'https://api.backup-example.com/data'
'https://api.backup-example.com/data',
];
try {
const response = await webRequest.requestMultiEndpoint(endpoints, {
method: 'GET'
method: 'GET',
});
const data = await response.json();
console.log(data);
@@ -133,7 +134,7 @@ async function customRequest() {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ custom: 'data' }),
timeoutMs: 10000 // Timeout in milliseconds
timeoutMs: 10000, // Timeout in milliseconds
});
if (response.ok) {
const result = await response.json();
@@ -153,14 +154,9 @@ customRequest();
`@push.rocks/webrequest` offers a streamlined, secure way to handle web requests from browsers, supporting various HTTP methods, response caching, and requests to multiple endpoints with fault tolerance. Its TypeScript integration ensures type safety and enhances developer productivity by enabling IntelliSense in supported IDEs.
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.

60
test/test.all.ts Normal file
View File

@@ -0,0 +1,60 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { webrequest } from '../ts/index.js';
// Simple smoke tests for v4 API
// Test 1: Basic fetch-compatible API
tap.test('should work as fetch replacement', async () => {
const response = await webrequest('https://httpbin.org/get', {
method: 'GET',
});
expect(response).toBeInstanceOf(Response);
expect(response.ok).toEqual(true);
console.log('API response status:', response.status);
});
// Test 2: JSON convenience method
tap.test('should support getJson convenience method', async () => {
interface HttpBinResponse {
url: string;
headers: Record<string, string>;
}
const data = await webrequest.getJson<HttpBinResponse>('https://httpbin.org/get');
console.log('getJson url:', data.url);
expect(data).toHaveProperty('url');
expect(data).toHaveProperty('headers');
});
// Test 3: POST with JSON
tap.test('should support postJson', async () => {
interface PostResponse {
json: any;
url: string;
}
const data = await webrequest.postJson<PostResponse>(
'https://httpbin.org/post',
{ test: 'data' }
);
expect(data).toHaveProperty('url');
expect(data).toHaveProperty('json');
console.log('postJson works');
});
// Test 4: Caching
tap.test('should support caching', async () => {
const data1 = await webrequest.getJson('https://httpbin.org/get', {
cacheStrategy: 'cache-first'
});
const data2 = await webrequest.getJson('https://httpbin.org/get', {
cacheStrategy: 'cache-first'
});
expect(data1).toHaveProperty('url');
expect(data2).toHaveProperty('url');
console.log('Caching works');
});
export default tap.start();

View File

@@ -1,11 +0,0 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as webrequest from '../ts/index.js';
tap.test('should run multiendpoint request', async (tools) => {
const response = await new webrequest.WebRequest().request('https://api.signup.software', {
method: 'GET',
});
console.log(JSON.stringify(await response.text()));
});
tap.start();

View File

@@ -1,5 +1,5 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as webrequest from '../ts/index.js';
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { webrequest } from '../ts/index.js';
// test dependencies
import * as typedserver from '@api.global/typedserver';
@@ -18,7 +18,7 @@ tap.test('setup test server', async () => {
new typedserver.servertools.Handler('GET', (req, res) => {
res.status(429);
res.end();
})
}),
);
testServer.addRoute(
@@ -26,7 +26,7 @@ tap.test('setup test server', async () => {
new typedserver.servertools.Handler('GET', (req, res) => {
res.status(500);
res.end();
})
}),
);
testServer.addRoute(
@@ -36,43 +36,62 @@ tap.test('setup test server', async () => {
res.send({
hithere: 'hi',
});
})
}),
);
await testServer.start();
});
tap.test('first test', async (tools) => {
const response = await (
await new webrequest.WebRequest().requestMultiEndpoint(
[
'http://localhost:2345/apiroute1',
tap.test('should handle fallback URLs', async () => {
const response = await webrequest(
'http://localhost:2345/apiroute1',
{
fallbackUrls: [
'http://localhost:2345/apiroute2',
'http://localhost:2345/apiroute4',
'http://localhost:2345/apiroute3',
],
{
method: 'GET',
}
)
).json();
retry: {
maxAttempts: 3,
backoff: 'constant',
initialDelay: 100,
},
}
);
const response2 = await new webrequest.WebRequest().getJson('http://localhost:2345/apiroute3');
const data = await response.json();
console.log('response with fallbacks: ' + JSON.stringify(data));
expect(data).toHaveProperty('hithere');
});
console.log('response 1: ' + JSON.stringify(response));
console.log('response 2: ' + JSON.stringify(response2));
expect(response).toHaveProperty('hithere'); //.to.equal('hi');
expect(response2).toHaveProperty('hithere'); //.to.equal('hi');
tap.test('should use getJson convenience method', async () => {
const data = await webrequest.getJson('http://localhost:2345/apiroute3');
console.log('getJson response: ' + JSON.stringify(data));
expect(data).toHaveProperty('hithere');
});
tap.test('should cache response', async () => {
const webrequestInstance = new webrequest.WebRequest();
const response = await webrequestInstance.getJson('http://localhost:2345/apiroute3', true);
expect(response).toHaveProperty('hithere');
// First request - goes to network
const response1 = await webrequest.getJson(
'http://localhost:2345/apiroute3',
{
cacheStrategy: 'cache-first',
}
);
expect(response1).toHaveProperty('hithere');
// Stop server
await testServer.stop();
const response2 = await webrequestInstance.getJson('http://localhost:2345/apiroute3', true);
// Second request - should use cache since server is down
const response2 = await webrequest.getJson(
'http://localhost:2345/apiroute3',
{
cacheStrategy: 'network-first', // Will fallback to cache on network error
}
);
expect(response2).toHaveProperty('hithere');
console.log('Cache fallback worked');
});
tap.start();
export default tap.start();

312
test/test.v4.node.ts Normal file
View File

@@ -0,0 +1,312 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { webrequest, WebrequestClient } from '../ts/index.js';
import * as typedserver from '@api.global/typedserver';
let testServer: typedserver.servertools.Server;
// Setup test server
tap.test('setup test server for v4 tests', async () => {
testServer = new typedserver.servertools.Server({
cors: false,
forceSsl: false,
port: 2346,
});
// Route that returns JSON with cache headers
testServer.addRoute(
'/cached',
new typedserver.servertools.Handler('GET', (req, res) => {
res.setHeader('Cache-Control', 'max-age=60');
res.setHeader('ETag', '"12345"');
res.status(200);
res.send({ data: 'cached response', timestamp: Date.now() });
}),
);
// Route that returns different data each time
testServer.addRoute(
'/dynamic',
new typedserver.servertools.Handler('GET', (req, res) => {
res.status(200);
res.send({ data: 'dynamic response', timestamp: Date.now() });
}),
);
// Route that sometimes fails (for retry testing)
let requestCount = 0;
testServer.addRoute(
'/flaky',
new typedserver.servertools.Handler('GET', (req, res) => {
requestCount++;
if (requestCount < 3) {
res.status(500);
res.end();
} else {
res.status(200);
res.send({ success: true, attempts: requestCount });
}
}),
);
// Route that returns 304 when ETag matches
testServer.addRoute(
'/conditional',
new typedserver.servertools.Handler('GET', (req, res) => {
const ifNoneMatch = req.headers['if-none-match'];
if (ifNoneMatch === '"67890"') {
res.status(304);
res.end();
} else {
res.setHeader('ETag', '"67890"');
res.status(200);
res.send({ data: 'conditional response' });
}
}),
);
// POST route for testing
testServer.addRoute(
'/post',
new typedserver.servertools.Handler('POST', (req, res) => {
res.status(200);
res.send({ received: true });
}),
);
await testServer.start();
});
// Test 1: Basic fetch-compatible API
tap.test('should work as fetch replacement', async () => {
const response = await webrequest('http://localhost:2346/dynamic');
expect(response).toBeInstanceOf(Response);
expect(response.ok).toEqual(true);
expect(response.status).toEqual(200);
const data = await response.json();
expect(data).toHaveProperty('data');
expect(data.data).toEqual('dynamic response');
});
// Test 2: getJson convenience method with generics
tap.test('should support getJson with type safety', async () => {
interface TestData {
data: string;
timestamp: number;
}
const data = await webrequest.getJson<TestData>('http://localhost:2346/dynamic');
expect(data).toHaveProperty('data');
expect(data).toHaveProperty('timestamp');
expect(typeof data.timestamp).toEqual('number');
});
// Test 3: POST with JSON
tap.test('should support postJson', async () => {
const data = await webrequest.postJson('http://localhost:2346/post', {
test: 'data'
});
expect(data).toHaveProperty('received');
expect(data.received).toEqual(true);
});
// Test 4: Cache strategy - network-first
tap.test('should support network-first cache strategy', async () => {
const response1 = await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'network-first'
});
const data1 = await response1.json();
expect(data1).toHaveProperty('timestamp');
// Second request should hit network but may use cache on error
const response2 = await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'network-first'
});
const data2 = await response2.json();
expect(data2).toHaveProperty('timestamp');
});
// Test 5: Cache strategy - cache-first
tap.test('should support cache-first strategy', async () => {
// First request goes to network
const response1 = await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'cache-first'
});
const data1 = await response1.json();
const timestamp1 = data1.timestamp;
// Second request should use cache
const response2 = await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'cache-first'
});
const data2 = await response2.json();
// Timestamps should be identical if cached
expect(data2.timestamp).toEqual(timestamp1);
});
// Test 6: Retry system
tap.test('should retry failed requests', async () => {
const response = await webrequest('http://localhost:2346/flaky', {
retry: {
maxAttempts: 3,
backoff: 'constant',
initialDelay: 100
}
});
const data = await response.json();
expect(data.success).toEqual(true);
expect(data.attempts).toBeGreaterThanOrEqual(3);
});
// Test 7: Fallback URLs
tap.test('should support fallback URLs', async () => {
const response = await webrequest('http://localhost:2346/nonexistent', {
fallbackUrls: ['http://localhost:2346/dynamic'],
retry: {
maxAttempts: 2
}
});
expect(response.ok).toEqual(true);
const data = await response.json();
expect(data).toHaveProperty('data');
});
// Test 8: Request interceptors
tap.test('should support request interceptors', async () => {
let interceptorCalled = false;
const response = await webrequest('http://localhost:2346/dynamic', {
interceptors: {
request: [(req) => {
interceptorCalled = true;
const headers = new Headers(req.headers);
headers.set('X-Custom-Header', 'test');
return new Request(req, { headers });
}]
}
});
expect(interceptorCalled).toEqual(true);
expect(response.ok).toEqual(true);
});
// Test 9: Response interceptors
tap.test('should support response interceptors', async () => {
let interceptorCalled = false;
let capturedStatus: number;
const response = await webrequest('http://localhost:2346/dynamic', {
interceptors: {
response: [(res) => {
interceptorCalled = true;
capturedStatus = res.status;
return res;
}]
}
});
expect(interceptorCalled).toEqual(true);
expect(capturedStatus!).toEqual(200);
});
// Test 10: Global interceptors with WebrequestClient
tap.test('should support global interceptors', async () => {
const client = new WebrequestClient();
let globalInterceptorCalled = false;
client.addRequestInterceptor((req) => {
globalInterceptorCalled = true;
return req;
});
await client.request('http://localhost:2346/dynamic');
expect(globalInterceptorCalled).toEqual(true);
});
// Test 11: Request deduplication
tap.test('should deduplicate simultaneous requests', async () => {
const start = Date.now();
// Make 3 identical requests simultaneously
const [res1, res2, res3] = await Promise.all([
webrequest('http://localhost:2346/dynamic', { deduplicate: true }),
webrequest('http://localhost:2346/dynamic', { deduplicate: true }),
webrequest('http://localhost:2346/dynamic', { deduplicate: true }),
]);
const [data1, data2, data3] = await Promise.all([
res1.json(),
res2.json(),
res3.json(),
]);
// All should have the same timestamp (same response)
expect(data1.timestamp).toEqual(data2.timestamp);
expect(data2.timestamp).toEqual(data3.timestamp);
});
// Test 12: Timeout
tap.test('should support timeout', async () => {
try {
await webrequest('http://localhost:2346/dynamic', {
timeout: 1 // 1ms timeout should fail
});
throw new Error('Should have timed out');
} catch (error) {
expect(error.message).toContain('timeout');
}
});
// Test 13: WebrequestClient with default options
tap.test('should support WebrequestClient with defaults', async () => {
const client = new WebrequestClient({
logging: false,
cacheStrategy: 'network-first',
timeout: 30000
});
const data = await client.getJson('http://localhost:2346/dynamic');
expect(data).toHaveProperty('data');
});
// Test 14: Clear cache
tap.test('should clear cache', async () => {
// Cache a request
await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'cache-first'
});
// Clear cache
await webrequest.clearCache();
// This should work even though cache is cleared
const response = await webrequest('http://localhost:2346/cached', {
cacheStrategy: 'cache-first'
});
expect(response.ok).toEqual(true);
});
// Test 15: Backward compatibility with WebRequest class
tap.test('should maintain backward compatibility with v3 API', async () => {
const { WebRequest } = await import('../ts/index.js');
const client = new WebRequest({ logging: false });
const data = await client.getJson('http://localhost:2346/dynamic');
expect(data).toHaveProperty('data');
// Test POST
const postData = await client.postJson('http://localhost:2346/post', {
test: 'data'
});
expect(postData).toHaveProperty('received');
});
// Cleanup
tap.test('stop test server', async () => {
await testServer.stop();
});
export default tap.start();

View File

@@ -4,5 +4,6 @@
export const commitinfo = {
name: '@push.rocks/webrequest',
version: '3.0.37',
description: 'A module for making secure web requests from browsers with support for caching and fault tolerance.'
}
description:
'A module for making secure web requests from browsers with support for caching and fault tolerance.',
};

172
ts/cache/cache.headers.ts vendored Normal file
View File

@@ -0,0 +1,172 @@
/**
* HTTP Cache Header parsing and utilities
* Implements RFC 7234 (HTTP Caching)
*/
import type { ICacheMetadata } from '../webrequest.types.js';
/**
* Parse Cache-Control header into metadata
*/
export function parseCacheControl(
cacheControlHeader: string | null,
): Partial<ICacheMetadata> {
const metadata: Partial<ICacheMetadata> = {
maxAge: 0,
immutable: false,
noCache: false,
noStore: false,
mustRevalidate: false,
};
if (!cacheControlHeader) {
return metadata;
}
const directives = cacheControlHeader
.toLowerCase()
.split(',')
.map((d) => d.trim());
for (const directive of directives) {
if (directive === 'no-cache') {
metadata.noCache = true;
} else if (directive === 'no-store') {
metadata.noStore = true;
} else if (directive === 'immutable') {
metadata.immutable = true;
} else if (directive === 'must-revalidate') {
metadata.mustRevalidate = true;
} else if (directive.startsWith('max-age=')) {
const maxAge = parseInt(directive.split('=')[1], 10);
if (!isNaN(maxAge)) {
metadata.maxAge = maxAge * 1000; // Convert to milliseconds
}
}
}
return metadata;
}
/**
* Parse Expires header into timestamp
*/
export function parseExpires(expiresHeader: string | null): number | undefined {
if (!expiresHeader) {
return undefined;
}
try {
const date = new Date(expiresHeader);
return date.getTime();
} catch {
return undefined;
}
}
/**
* Extract cache metadata from response headers
*/
export function extractCacheMetadata(headers: Headers): ICacheMetadata {
const cacheControl = headers.get('cache-control');
const expires = headers.get('expires');
const etag = headers.get('etag');
const lastModified = headers.get('last-modified');
const metadata = parseCacheControl(cacheControl);
// If no max-age from Cache-Control, try Expires header
if (metadata.maxAge === 0 && expires) {
const expiresTime = parseExpires(expires);
if (expiresTime) {
metadata.maxAge = Math.max(0, expiresTime - Date.now());
}
}
return {
maxAge: metadata.maxAge || 0,
etag: etag || undefined,
lastModified: lastModified || undefined,
immutable: metadata.immutable || false,
noCache: metadata.noCache || false,
noStore: metadata.noStore || false,
mustRevalidate: metadata.mustRevalidate || false,
};
}
/**
* Check if a cached response is still fresh
*/
export function isFresh(
cacheEntry: { timestamp: number; maxAge?: number },
metadata: ICacheMetadata,
): boolean {
// no-store means never cache
if (metadata.noStore) {
return false;
}
// If immutable, it's always fresh
if (metadata.immutable) {
return true;
}
const age = Date.now() - cacheEntry.timestamp;
const maxAge = cacheEntry.maxAge || metadata.maxAge || 0;
// If no max-age specified, consider stale
if (maxAge === 0) {
return false;
}
return age < maxAge;
}
/**
* Check if revalidation is required
*/
export function requiresRevalidation(metadata: ICacheMetadata): boolean {
return metadata.noCache || metadata.mustRevalidate;
}
/**
* Create conditional request headers for revalidation
*/
export function createConditionalHeaders(cacheEntry: {
etag?: string;
lastModified?: string;
}): HeadersInit {
const headers: Record<string, string> = {};
if (cacheEntry.etag) {
headers['if-none-match'] = cacheEntry.etag;
}
if (cacheEntry.lastModified) {
headers['if-modified-since'] = cacheEntry.lastModified;
}
return headers;
}
/**
* Convert Headers object to plain object for storage
*/
export function headersToObject(headers: Headers): Record<string, string> {
const obj: Record<string, string> = {};
headers.forEach((value, key) => {
obj[key] = value;
});
return obj;
}
/**
* Convert plain object back to Headers
*/
export function objectToHeaders(obj: Record<string, string>): Headers {
const headers = new Headers();
Object.entries(obj).forEach(([key, value]) => {
headers.set(key, value);
});
return headers;
}

156
ts/cache/cache.manager.ts vendored Normal file
View File

@@ -0,0 +1,156 @@
/**
* Cache manager - orchestrates caching logic
*/
import type {
ICacheOptions,
TCacheStrategy,
TStandardCacheMode,
} from '../webrequest.types.js';
import { CacheStore } from './cache.store.js';
import {
getStrategyHandler,
type IStrategyContext,
type IStrategyResult,
} from './cache.strategies.js';
import { extractCacheMetadata } from './cache.headers.js';
export class CacheManager {
private cacheStore: CacheStore;
constructor(dbName?: string, storeName?: string) {
this.cacheStore = new CacheStore(dbName, storeName);
}
/**
* Execute a request with caching
*/
public async execute(
request: Request,
options: ICacheOptions & { logging?: boolean },
fetchFn: (request: Request) => Promise<Response>,
): Promise<IStrategyResult> {
// Determine the cache strategy
const strategy = this.determineStrategy(request, options);
// If no caching (no-store or network-only), bypass cache
if (strategy === 'network-only') {
const response = await fetchFn(request);
return {
response,
fromCache: false,
revalidated: false,
};
}
// Generate cache key
const cacheKey = this.generateCacheKey(request, options);
// Get strategy handler
const handler = getStrategyHandler(strategy);
// Execute strategy
const context: IStrategyContext = {
request,
cacheKey,
cacheStore: this.cacheStore,
fetchFn,
logging: options.logging,
};
return await handler.execute(context);
}
/**
* Determine the caching strategy based on options and request
*/
private determineStrategy(
request: Request,
options: ICacheOptions,
): TCacheStrategy {
// If explicit strategy provided, use it
if (options.cacheStrategy) {
return options.cacheStrategy;
}
// Map standard cache modes to strategies
if (options.cache) {
return this.mapCacheModeToStrategy(options.cache);
}
// Check request cache mode
if (request.cache) {
return this.mapCacheModeToStrategy(request.cache as TStandardCacheMode);
}
// Default strategy
return 'network-first';
}
/**
* Map standard fetch cache modes to our strategies
*/
private mapCacheModeToStrategy(
cacheMode: TStandardCacheMode,
): TCacheStrategy {
switch (cacheMode) {
case 'default':
return 'network-first';
case 'no-store':
case 'reload':
return 'network-only';
case 'no-cache':
return 'network-first'; // Will use revalidation
case 'force-cache':
return 'cache-first';
case 'only-if-cached':
return 'cache-only';
default:
return 'network-first';
}
}
/**
* Generate cache key
*/
private generateCacheKey(request: Request, options: ICacheOptions): string {
// If custom cache key provided
if (options.cacheKey) {
if (typeof options.cacheKey === 'function') {
return options.cacheKey(request);
}
return options.cacheKey;
}
// Default cache key generation
return this.cacheStore.generateCacheKey(request);
}
/**
* Clear the cache
*/
public async clear(): Promise<void> {
await this.cacheStore.clear();
}
/**
* Delete a specific cache entry
*/
public async delete(cacheKey: string): Promise<void> {
await this.cacheStore.delete(cacheKey);
}
/**
* Check if a cache entry exists
*/
public async has(cacheKey: string): Promise<boolean> {
return await this.cacheStore.has(cacheKey);
}
/**
* Get the underlying cache store
*/
public getStore(): CacheStore {
return this.cacheStore;
}
}

154
ts/cache/cache.store.ts vendored Normal file
View File

@@ -0,0 +1,154 @@
/**
* Cache storage layer using IndexedDB via @push.rocks/webstore
*/
import * as plugins from '../webrequest.plugins.js';
import type { ICacheEntry } from '../webrequest.types.js';
export class CacheStore {
private webstore: plugins.webstore.WebStore;
private initPromise: Promise<void>;
constructor(dbName: string = 'webrequest-v4', storeName: string = 'cache') {
this.webstore = new plugins.webstore.WebStore({
dbName,
storeName,
});
// Initialize the store
this.initPromise = this.init();
}
/**
* Initialize the store
*/
private async init(): Promise<void> {
// WebStore handles initialization internally
// This method exists for future extension if needed
}
/**
* Generate a cache key from a request
*/
public generateCacheKey(request: Request): string {
// Use URL + method as the base key
const url = request.url;
const method = request.method;
// For GET requests, just use the URL
if (method === 'GET') {
return url;
}
// For other methods, include the method
return `${method}:${url}`;
}
/**
* Store a response in the cache
*/
public async set(cacheKey: string, entry: ICacheEntry): Promise<void> {
await this.initPromise;
await this.webstore.set(cacheKey, entry);
}
/**
* Retrieve a cached response
*/
public async get(cacheKey: string): Promise<ICacheEntry | null> {
await this.initPromise;
try {
const entry = (await this.webstore.get(cacheKey)) as ICacheEntry;
return entry || null;
} catch (error) {
// If entry doesn't exist or is corrupted, return null
return null;
}
}
/**
* Check if a cache entry exists
*/
public async has(cacheKey: string): Promise<boolean> {
await this.initPromise;
return await this.webstore.check(cacheKey);
}
/**
* Delete a cache entry
*/
public async delete(cacheKey: string): Promise<void> {
await this.initPromise;
await this.webstore.delete(cacheKey);
}
/**
* Clear all cache entries
*/
public async clear(): Promise<void> {
await this.initPromise;
await this.webstore.clear();
}
/**
* Create a Response object from a cache entry
*/
public responseFromCacheEntry(entry: ICacheEntry): Response {
const headers = new Headers(entry.headers);
return new Response(entry.response, {
status: entry.status,
statusText: entry.statusText,
headers,
});
}
/**
* Create a cache entry from a Response object
*/
public async cacheEntryFromResponse(
url: string,
response: Response,
metadata?: { maxAge?: number; etag?: string; lastModified?: string },
): Promise<ICacheEntry> {
// Clone the response so we can read it multiple times
const clonedResponse = response.clone();
const buffer = await clonedResponse.arrayBuffer();
// Extract headers
const headers: Record<string, string> = {};
clonedResponse.headers.forEach((value, key) => {
headers[key] = value;
});
return {
response: buffer,
headers,
timestamp: Date.now(),
etag: metadata?.etag || clonedResponse.headers.get('etag') || undefined,
lastModified:
metadata?.lastModified ||
clonedResponse.headers.get('last-modified') ||
undefined,
maxAge: metadata?.maxAge,
url,
status: clonedResponse.status,
statusText: clonedResponse.statusText,
};
}
/**
* Prune expired entries (garbage collection)
* Returns the number of entries deleted
*/
public async pruneExpired(): Promise<number> {
await this.initPromise;
// Note: WebStore doesn't provide a way to list all keys
// This would need to be implemented if we want automatic cleanup
// For now, we rely on individual entry checks
return 0;
}
}

377
ts/cache/cache.strategies.ts vendored Normal file
View File

@@ -0,0 +1,377 @@
/**
* Cache strategy implementations
*/
import type {
ICacheEntry,
ICacheMetadata,
TCacheStrategy,
} from '../webrequest.types.js';
import { CacheStore } from './cache.store.js';
import {
extractCacheMetadata,
isFresh,
requiresRevalidation,
createConditionalHeaders,
headersToObject,
} from './cache.headers.js';
export interface IStrategyContext {
request: Request;
cacheKey: string;
cacheStore: CacheStore;
fetchFn: (request: Request) => Promise<Response>;
logging?: boolean;
}
export interface IStrategyResult {
response: Response;
fromCache: boolean;
revalidated: boolean;
}
/**
* Base strategy handler interface
*/
export interface ICacheStrategyHandler {
execute(context: IStrategyContext): Promise<IStrategyResult>;
}
/**
* Network-First Strategy
* Try network first, fallback to cache on failure
*/
export class NetworkFirstStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
try {
// Try network first
const response = await context.fetchFn(context.request);
// If successful, cache it
if (response.ok) {
await this.cacheResponse(context, response);
}
return {
response,
fromCache: false,
revalidated: false,
};
} catch (error) {
// Network failed, try cache
if (context.logging) {
console.log('[webrequest] Network failed, trying cache:', error);
}
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
// No cache available, re-throw error
throw error;
}
}
private async cacheResponse(
context: IStrategyContext,
response: Response,
): Promise<void> {
const metadata = extractCacheMetadata(response.headers);
// Don't cache if no-store
if (metadata.noStore) {
return;
}
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
}
/**
* Cache-First Strategy
* Check cache first, fetch if miss or stale
*/
export class CacheFirstStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
// Check cache first
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
const metadata = extractCacheMetadata(new Headers(cachedEntry.headers));
// Check if cache is fresh
if (isFresh(cachedEntry, metadata)) {
if (context.logging) {
console.log('[webrequest] Cache hit (fresh):', context.request.url);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
// If requires revalidation, check with server
if (
requiresRevalidation(metadata) &&
(cachedEntry.etag || cachedEntry.lastModified)
) {
return await this.revalidate(context, cachedEntry);
}
}
// Cache miss or stale, fetch from network
if (context.logging) {
console.log('[webrequest] Cache miss, fetching:', context.request.url);
}
const response = await context.fetchFn(context.request);
// Cache the response
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
return {
response,
fromCache: false,
revalidated: false,
};
}
private async revalidate(
context: IStrategyContext,
cachedEntry: ICacheEntry,
): Promise<IStrategyResult> {
const conditionalHeaders = createConditionalHeaders(cachedEntry);
// Create a new request with conditional headers
const revalidateRequest = new Request(context.request.url, {
method: context.request.method,
headers: {
...headersToObject(context.request.headers),
...conditionalHeaders,
},
});
try {
const response = await context.fetchFn(revalidateRequest);
// 304 Not Modified - cache is still valid
if (response.status === 304) {
if (context.logging) {
console.log(
'[webrequest] Cache revalidated (304):',
context.request.url,
);
}
// Update timestamp
cachedEntry.timestamp = Date.now();
await context.cacheStore.set(context.cacheKey, cachedEntry);
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: true,
};
}
// Response changed, cache the new one
if (response.ok) {
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
}
return {
response,
fromCache: false,
revalidated: true,
};
} catch (error) {
// Revalidation failed, use cached response
if (context.logging) {
console.log('[webrequest] Revalidation failed, using cache:', error);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
}
}
/**
* Stale-While-Revalidate Strategy
* Return cache immediately, update in background
*/
export class StaleWhileRevalidateStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (cachedEntry) {
// Return cached response immediately
const cachedResponse =
context.cacheStore.responseFromCacheEntry(cachedEntry);
// Revalidate in background
this.revalidateInBackground(context, cachedEntry).catch((error) => {
if (context.logging) {
console.warn('[webrequest] Background revalidation failed:', error);
}
});
return {
response: cachedResponse,
fromCache: true,
revalidated: false,
};
}
// No cache, fetch from network
const response = await context.fetchFn(context.request);
// Cache the response
const metadata = extractCacheMetadata(response.headers);
if (!metadata.noStore && response.ok) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
metadata,
);
await context.cacheStore.set(context.cacheKey, entry);
}
return {
response,
fromCache: false,
revalidated: false,
};
}
private async revalidateInBackground(
context: IStrategyContext,
cachedEntry: ICacheEntry,
): Promise<void> {
const metadata = extractCacheMetadata(new Headers(cachedEntry.headers));
// Check if revalidation is needed
if (isFresh(cachedEntry, metadata) && !requiresRevalidation(metadata)) {
return;
}
try {
const response = await context.fetchFn(context.request);
if (response.ok) {
const newMetadata = extractCacheMetadata(response.headers);
if (!newMetadata.noStore) {
const entry = await context.cacheStore.cacheEntryFromResponse(
context.request.url,
response,
newMetadata,
);
await context.cacheStore.set(context.cacheKey, entry);
if (context.logging) {
console.log(
'[webrequest] Background revalidation complete:',
context.request.url,
);
}
}
}
} catch (error) {
// Background revalidation failed, keep existing cache
if (context.logging) {
console.warn('[webrequest] Background revalidation failed:', error);
}
}
}
}
/**
* Network-Only Strategy
* Never use cache
*/
export class NetworkOnlyStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const response = await context.fetchFn(context.request);
return {
response,
fromCache: false,
revalidated: false,
};
}
}
/**
* Cache-Only Strategy
* Only use cache, fail if miss
*/
export class CacheOnlyStrategy implements ICacheStrategyHandler {
async execute(context: IStrategyContext): Promise<IStrategyResult> {
const cachedEntry = await context.cacheStore.get(context.cacheKey);
if (!cachedEntry) {
throw new Error(
`Cache miss for ${context.request.url} (cache-only mode)`,
);
}
return {
response: context.cacheStore.responseFromCacheEntry(cachedEntry),
fromCache: true,
revalidated: false,
};
}
}
/**
* Get strategy handler for a given strategy type
*/
export function getStrategyHandler(
strategy: TCacheStrategy,
): ICacheStrategyHandler {
switch (strategy) {
case 'network-first':
return new NetworkFirstStrategy();
case 'cache-first':
return new CacheFirstStrategy();
case 'stale-while-revalidate':
return new StaleWhileRevalidateStrategy();
case 'network-only':
return new NetworkOnlyStrategy();
case 'cache-only':
return new CacheOnlyStrategy();
default:
return new NetworkFirstStrategy();
}
}

View File

@@ -1,220 +1,47 @@
import * as plugins from './webrequest.plugins.js';
export interface IWebrequestContructorOptions {
logging?: boolean;
}
/**
* web request
* @push.rocks/webrequest v4
* Modern, fetch-compatible web request library with intelligent caching
*/
export class WebRequest {
public cacheStore = new plugins.webstore.WebStore({
dbName: 'webrequest',
storeName: 'webrequest',
});
// Main exports
export { webrequest } from './webrequest.function.js';
export { WebrequestClient } from './webrequest.client.js';
public options: IWebrequestContructorOptions;
// Type exports
export type {
IWebrequestOptions,
ICacheOptions,
IRetryOptions,
IInterceptors,
TCacheStrategy,
TStandardCacheMode,
TBackoffStrategy,
TWebrequestResult,
IWebrequestSuccess,
IWebrequestError,
ICacheEntry,
ICacheMetadata,
} from './webrequest.types.js';
constructor(public optionsArg: IWebrequestContructorOptions = {}) {
this.options = {
logging: true,
...optionsArg,
};
}
export type {
TRequestInterceptor,
TResponseInterceptor,
TErrorInterceptor,
} from './interceptors/interceptor.types.js';
public async getJson(urlArg: string, useCacheArg: boolean = false) {
const response: Response = await this.request(urlArg, {
method: 'GET',
useCache: useCacheArg,
});
const responseText = await response.text();
const responseResult = plugins.smartjson.parse(responseText);
return responseResult;
}
// Advanced exports for custom implementations
export { CacheManager } from './cache/cache.manager.js';
export { CacheStore } from './cache/cache.store.js';
export { RetryManager } from './retry/retry.manager.js';
export { InterceptorManager } from './interceptors/interceptor.manager.js';
export { RequestDeduplicator } from './utils/deduplicator.js';
/**
* postJson
*/
public async postJson(urlArg: string, requestBody?: any, useCacheArg: boolean = false) {
const response: Response = await this.request(urlArg, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: plugins.smartjson.stringify(requestBody),
useCache: useCacheArg,
});
const responseText = await response.text();
const responseResult = plugins.smartjson.parse(responseText);
return responseResult;
}
/**
* put js
*/
public async putJson(urlArg: string, requestBody?: any, useStoreAsFallback: boolean = false) {
const response: Response = await this.request(urlArg, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: plugins.smartjson.stringify(requestBody),
});
const responseText = await response.text();
const responseResult = plugins.smartjson.parse(responseText);
return responseResult;
}
/**
* put js
*/
public async deleteJson(urlArg: string, useStoreAsFallback: boolean = false) {
const response: Response = await this.request(urlArg, {
headers: {
'Content-Type': 'application/json',
},
method: 'GET',
});
const responseText = await response.text();
const responseResult = plugins.smartjson.parse(responseText);
return responseResult;
}
public async request(
urlArg: string,
optionsArg: {
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
body?: any;
headers?: HeadersInit;
useCache?: boolean;
timeoutMs?: number;
}
) {
optionsArg = {
timeoutMs: 60000,
useCache: false,
...optionsArg,
};
let controller = new AbortController();
plugins.smartdelay.delayFor(optionsArg.timeoutMs).then(() => {
controller.abort();
});
let cachedResponseDeferred = plugins.smartpromise.defer<Response>();
let cacheUsed = false;
if (optionsArg.useCache && (await this.cacheStore.check(urlArg))) {
const responseBuffer: ArrayBuffer = await this.cacheStore.get(urlArg);
cachedResponseDeferred.resolve(new Response(responseBuffer, {}));
} else {
cachedResponseDeferred.resolve(null);
}
let response: Response = await fetch(urlArg, {
signal: controller.signal,
method: optionsArg.method,
headers: {
...(optionsArg.headers || {}),
},
body: optionsArg.body,
})
.catch(async (err) => {
if (optionsArg.useCache && (await cachedResponseDeferred.promise)) {
cacheUsed = true;
const cachedResponse = cachedResponseDeferred.promise;
return cachedResponse;
} else {
return err;
}
});
if (optionsArg.useCache && (await cachedResponseDeferred.promise) && response.status === 500) {
cacheUsed = true;
response = await cachedResponseDeferred.promise;
}
if (!cacheUsed && optionsArg.useCache && response.status < 300) {
const buffer = await response.clone().arrayBuffer();
await this.cacheStore.set(urlArg, buffer);
}
this.log(`${urlArg} answers with status: ${response.status}`);
return response;
}
/**
* a multi endpoint, fault tolerant request function
*/
public async requestMultiEndpoint(
urlArg: string | string[],
optionsArg: {
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
body?: any;
headers?: HeadersInit;
}
): Promise<Response> {
let allUrls: string[];
let usedUrlIndex = 0;
// determine what we got
if (Array.isArray(urlArg)) {
allUrls = urlArg;
} else {
allUrls = [urlArg];
}
const requestHistory: string[] = []; // keep track of the request history
const doHistoryCheck = async (
// check history for a
historyEntryTypeArg: string
) => {
requestHistory.push(historyEntryTypeArg);
if (historyEntryTypeArg === '429') {
console.log('got 429, so waiting a little bit.');
await plugins.smartdelay.delayFor(Math.floor(Math.random() * (2000 - 1000 + 1)) + 1000); // wait between 1 and 10 seconds
}
let numOfHistoryType = 0;
for (const entry of requestHistory) {
if (entry === historyEntryTypeArg) numOfHistoryType++;
}
if (numOfHistoryType > 2 * allUrls.length * usedUrlIndex) {
usedUrlIndex++;
}
};
// lets go recursive
const doRequest = async (urlToUse: string): Promise<any> => {
if (!urlToUse) {
throw new Error('request failed permanently');
}
this.log(`Getting ${urlToUse} with method ${optionsArg.method}`);
const response = await fetch(urlToUse, {
method: optionsArg.method,
headers: {
'Content-Type': 'application/json',
...(optionsArg.headers || {}),
},
body: optionsArg.body,
});
this.log(`${urlToUse} answers with status: ${response.status}`);
if (response.status >= 200 && response.status < 300) {
return response;
} else {
// lets perform a history check to determine failed urls
await doHistoryCheck(response.status.toString());
// lets fire the request
const result = await doRequest(allUrls[usedUrlIndex]);
return result;
}
};
const finalResponse: Response = await doRequest(allUrls[usedUrlIndex]);
return finalResponse;
}
public log(logArg: string) {
if (this.options.logging) {
console.log(logArg);
}
}
}
// Cache utilities
export {
extractCacheMetadata,
isFresh,
requiresRevalidation,
createConditionalHeaders,
headersToObject,
objectToHeaders,
} from './cache/cache.headers.js';

View File

@@ -0,0 +1,149 @@
/**
* Interceptor manager for request/response transformation
*/
import type {
TRequestInterceptor,
TResponseInterceptor,
TErrorInterceptor,
} from './interceptor.types.js';
export class InterceptorManager {
private requestInterceptors: TRequestInterceptor[] = [];
private responseInterceptors: TResponseInterceptor[] = [];
private errorInterceptors: TErrorInterceptor[] = [];
/**
* Add a request interceptor
*/
public addRequestInterceptor(interceptor: TRequestInterceptor): void {
this.requestInterceptors.push(interceptor);
}
/**
* Add a response interceptor
*/
public addResponseInterceptor(interceptor: TResponseInterceptor): void {
this.responseInterceptors.push(interceptor);
}
/**
* Add an error interceptor
*/
public addErrorInterceptor(interceptor: TErrorInterceptor): void {
this.errorInterceptors.push(interceptor);
}
/**
* Remove a request interceptor
*/
public removeRequestInterceptor(interceptor: TRequestInterceptor): void {
const index = this.requestInterceptors.indexOf(interceptor);
if (index > -1) {
this.requestInterceptors.splice(index, 1);
}
}
/**
* Remove a response interceptor
*/
public removeResponseInterceptor(interceptor: TResponseInterceptor): void {
const index = this.responseInterceptors.indexOf(interceptor);
if (index > -1) {
this.responseInterceptors.splice(index, 1);
}
}
/**
* Remove an error interceptor
*/
public removeErrorInterceptor(interceptor: TErrorInterceptor): void {
const index = this.errorInterceptors.indexOf(interceptor);
if (index > -1) {
this.errorInterceptors.splice(index, 1);
}
}
/**
* Clear all interceptors
*/
public clearAll(): void {
this.requestInterceptors = [];
this.responseInterceptors = [];
this.errorInterceptors = [];
}
/**
* Process request through all request interceptors
*/
public async processRequest(request: Request): Promise<Request> {
let processedRequest = request;
for (const interceptor of this.requestInterceptors) {
try {
processedRequest = await interceptor(processedRequest);
} catch (error) {
// If interceptor throws, process through error interceptors
throw await this.processError(
error instanceof Error ? error : new Error(String(error)),
);
}
}
return processedRequest;
}
/**
* Process response through all response interceptors
*/
public async processResponse(response: Response): Promise<Response> {
let processedResponse = response;
for (const interceptor of this.responseInterceptors) {
try {
processedResponse = await interceptor(processedResponse);
} catch (error) {
// If interceptor throws, process through error interceptors
throw await this.processError(
error instanceof Error ? error : new Error(String(error)),
);
}
}
return processedResponse;
}
/**
* Process error through all error interceptors
*/
public async processError(error: Error): Promise<Error> {
let processedError = error;
for (const interceptor of this.errorInterceptors) {
try {
processedError = await interceptor(processedError);
} catch (newError) {
// If error interceptor throws, use the new error
processedError =
newError instanceof Error ? newError : new Error(String(newError));
}
}
return processedError;
}
/**
* Get count of registered interceptors
*/
public getInterceptorCounts(): {
request: number;
response: number;
error: number;
} {
return {
request: this.requestInterceptors.length,
response: this.responseInterceptors.length,
error: this.errorInterceptors.length,
};
}
}

View File

@@ -0,0 +1,31 @@
/**
* Interceptor type definitions
*/
/**
* Request interceptor
* Transforms the request before it's sent
*/
export type TRequestInterceptor = (
request: Request,
) => Request | Promise<Request>;
/**
* Response interceptor
* Transforms the response after it's received
*/
export type TResponseInterceptor = (
response: Response,
) => Response | Promise<Response>;
/**
* Error interceptor
* Handles errors during request/response processing
*/
export type TErrorInterceptor = (error: Error) => Error | Promise<Error>;
export interface IInterceptors {
request?: TRequestInterceptor[];
response?: TResponseInterceptor[];
error?: TErrorInterceptor[];
}

199
ts/retry/retry.manager.ts Normal file
View File

@@ -0,0 +1,199 @@
/**
* Retry manager for handling request retries
*/
import * as plugins from '../webrequest.plugins.js';
import type { IRetryOptions } from '../webrequest.types.js';
import { getBackoffCalculator, addJitter } from './retry.strategies.js';
export class RetryManager {
private options: Required<IRetryOptions>;
constructor(options: IRetryOptions = {}) {
this.options = {
maxAttempts: options.maxAttempts ?? 3,
backoff: options.backoff ?? 'exponential',
initialDelay: options.initialDelay ?? 1000,
maxDelay: options.maxDelay ?? 30000,
retryOn: options.retryOn ?? [408, 429, 500, 502, 503, 504],
onRetry: options.onRetry ?? (() => {}),
};
}
/**
* Execute a request with retry logic
*/
public async execute<T>(
executeFn: () => Promise<T>,
shouldRetryFn?: (error: any, attempt: number) => boolean,
): Promise<T> {
let lastError: Error;
let lastResponse: Response | undefined;
for (let attempt = 1; attempt <= this.options.maxAttempts; attempt++) {
try {
const result = await executeFn();
// Check if result is a Response and if we should retry based on status
if (result instanceof Response) {
if (this.shouldRetryResponse(result)) {
lastResponse = result;
// If this is the last attempt, return the failed response
if (attempt === this.options.maxAttempts) {
return result;
}
// Calculate delay and retry
const delay = this.calculateDelay(attempt);
this.options.onRetry(
attempt,
new Error(`HTTP ${result.status}`),
delay,
);
await this.delay(delay);
continue;
}
}
// Success
return result;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
// Check if we should retry
const shouldRetry = shouldRetryFn
? shouldRetryFn(error, attempt)
: this.shouldRetryError(error);
// If this is the last attempt or we shouldn't retry, throw
if (attempt === this.options.maxAttempts || !shouldRetry) {
throw lastError;
}
// Calculate delay and retry
const delay = this.calculateDelay(attempt);
this.options.onRetry(attempt, lastError, delay);
await this.delay(delay);
}
}
// This should never be reached, but TypeScript needs it
throw lastError! || new Error('Max retry attempts reached');
}
/**
* Execute with multiple fallback URLs
*/
public async executeWithFallbacks(
urls: string[],
requestInit: RequestInit,
fetchFn: (url: string, init: RequestInit) => Promise<Response>,
): Promise<Response> {
if (urls.length === 0) {
throw new Error('No URLs provided for fallback execution');
}
let lastError: Error | undefined;
const failedUrls: string[] = [];
for (const url of urls) {
try {
// Try the URL with retry logic
const response = await this.execute(async () => {
return await fetchFn(url, requestInit);
});
// If successful (status < 400), return
if (response.status < 400) {
return response;
}
// If 4xx client error (except 408 timeout), don't try other URLs
if (
response.status >= 400 &&
response.status < 500 &&
response.status !== 408
) {
return response;
}
// Server error or timeout, try next URL
failedUrls.push(url);
lastError = new Error(`Request failed with status ${response.status}`);
} catch (error) {
failedUrls.push(url);
lastError = error instanceof Error ? error : new Error(String(error));
}
}
// All URLs failed
throw new Error(
`All URLs failed: ${failedUrls.join(', ')}. Last error: ${lastError?.message || 'Unknown error'}`,
);
}
/**
* Check if we should retry based on response status
*/
private shouldRetryResponse(response: Response): boolean {
const retryOn = this.options.retryOn;
if (typeof retryOn === 'function') {
return retryOn(response);
}
if (Array.isArray(retryOn)) {
return retryOn.includes(response.status);
}
return false;
}
/**
* Check if we should retry based on error
*/
private shouldRetryError(error: any): boolean {
// Network errors should be retried
if (error instanceof TypeError && error.message.includes('fetch')) {
return true;
}
// Timeout errors should be retried
if (error.name === 'AbortError' || error.message.includes('timeout')) {
return true;
}
// If retryOn is a function, use it
const retryOn = this.options.retryOn;
if (typeof retryOn === 'function') {
return retryOn(undefined as any, error);
}
return false;
}
/**
* Calculate delay for next retry
*/
private calculateDelay(attempt: number): number {
const calculator = getBackoffCalculator(this.options.backoff);
const baseDelay = calculator.calculate(
attempt,
this.options.initialDelay,
this.options.maxDelay,
);
// Add jitter to prevent thundering herd
return addJitter(baseDelay);
}
/**
* Delay execution
*/
private async delay(ms: number): Promise<void> {
await plugins.smartdelay.delayFor(ms);
}
}

View File

@@ -0,0 +1,67 @@
/**
* Retry backoff strategies
*/
import type { TBackoffStrategy } from '../webrequest.types.js';
export interface IBackoffCalculator {
calculate(attempt: number, initialDelay: number, maxDelay: number): number;
}
/**
* Exponential backoff strategy
* Delay increases exponentially: initialDelay * 2^attempt
*/
export class ExponentialBackoff implements IBackoffCalculator {
calculate(attempt: number, initialDelay: number, maxDelay: number): number {
const delay = initialDelay * Math.pow(2, attempt - 1);
return Math.min(delay, maxDelay);
}
}
/**
* Linear backoff strategy
* Delay increases linearly: initialDelay * attempt
*/
export class LinearBackoff implements IBackoffCalculator {
calculate(attempt: number, initialDelay: number, maxDelay: number): number {
const delay = initialDelay * attempt;
return Math.min(delay, maxDelay);
}
}
/**
* Constant backoff strategy
* Delay stays constant: initialDelay
*/
export class ConstantBackoff implements IBackoffCalculator {
calculate(attempt: number, initialDelay: number, maxDelay: number): number {
return Math.min(initialDelay, maxDelay);
}
}
/**
* Get backoff calculator for a given strategy
*/
export function getBackoffCalculator(
strategy: TBackoffStrategy,
): IBackoffCalculator {
switch (strategy) {
case 'exponential':
return new ExponentialBackoff();
case 'linear':
return new LinearBackoff();
case 'constant':
return new ConstantBackoff();
default:
return new ExponentialBackoff();
}
}
/**
* Add jitter to delay to prevent thundering herd
*/
export function addJitter(delay: number, jitterFactor: number = 0.1): number {
const jitter = delay * jitterFactor * Math.random();
return delay + jitter;
}

105
ts/utils/deduplicator.ts Normal file
View File

@@ -0,0 +1,105 @@
/**
* Request deduplication system
* Prevents multiple simultaneous identical requests
*/
import * as plugins from '../webrequest.plugins.js';
export class RequestDeduplicator {
private inFlightRequests: Map<
string,
plugins.smartpromise.Deferred<Response>
> = new Map();
/**
* Generate a deduplication key from a request
*/
public generateKey(request: Request): string {
// Use URL + method as the base key
const url = request.url;
const method = request.method;
// For GET/HEAD requests, just use URL + method
if (method === 'GET' || method === 'HEAD') {
return `${method}:${url}`;
}
// For other methods, we can't deduplicate as easily
// (body might be different)
// Use a timestamp to make it unique
return `${method}:${url}:${Date.now()}`;
}
/**
* Execute a request with deduplication
*/
public async execute(
key: string,
executeFn: () => Promise<Response>,
): Promise<{ response: Response; wasDeduplicated: boolean }> {
// Check if request is already in flight
const existingDeferred = this.inFlightRequests.get(key);
if (existingDeferred) {
// Wait for the existing request to complete
const response = await existingDeferred.promise;
// Clone the response so it can be used multiple times
return {
response: response.clone(),
wasDeduplicated: true,
};
}
// Create a new deferred for this request
const deferred = plugins.smartpromise.defer<Response>();
this.inFlightRequests.set(key, deferred);
try {
// Execute the request
const response = await executeFn();
// Resolve the deferred
deferred.resolve(response);
// Clean up
this.inFlightRequests.delete(key);
// Return the original response
return {
response,
wasDeduplicated: false,
};
} catch (error) {
// Reject the deferred
deferred.reject(error);
// Clean up
this.inFlightRequests.delete(key);
// Re-throw the error
throw error;
}
}
/**
* Check if a request is currently in flight
*/
public isInFlight(key: string): boolean {
return this.inFlightRequests.has(key);
}
/**
* Get the number of in-flight requests
*/
public getInFlightCount(): number {
return this.inFlightRequests.size;
}
/**
* Clear all in-flight requests
*/
public clear(): void {
this.inFlightRequests.clear();
}
}

66
ts/utils/timeout.ts Normal file
View File

@@ -0,0 +1,66 @@
/**
* Timeout handling utilities
*/
import * as plugins from '../webrequest.plugins.js';
/**
* Create an AbortController with timeout
*/
export function createTimeoutController(timeoutMs: number): {
controller: AbortController;
cleanup: () => void;
} {
const controller = new AbortController();
let timeoutId: any;
// Set up timeout
plugins.smartdelay
.delayFor(timeoutMs)
.then(() => {
controller.abort();
})
.then((result) => {
timeoutId = result;
});
// Cleanup function to clear timeout
const cleanup = () => {
if (timeoutId !== undefined) {
// smartdelay doesn't expose a cancel method, so we just ensure
// the controller won't abort if already completed
}
};
return { controller, cleanup };
}
/**
* Execute a fetch with timeout
*/
export async function fetchWithTimeout(
url: string,
init: RequestInit,
timeoutMs: number,
): Promise<Response> {
const { controller, cleanup } = createTimeoutController(timeoutMs);
try {
const response = await fetch(url, {
...init,
signal: controller.signal,
});
cleanup();
return response;
} catch (error) {
cleanup();
// Re-throw with more informative error if it's a timeout
if (error instanceof Error && error.name === 'AbortError') {
throw new Error(`Request timeout after ${timeoutMs}ms: ${url}`);
}
throw error;
}
}

326
ts/webrequest.client.ts Normal file
View File

@@ -0,0 +1,326 @@
/**
* WebrequestClient - Advanced configuration and global interceptors
*/
import type { IWebrequestOptions } from './webrequest.types.js';
import type {
TRequestInterceptor,
TResponseInterceptor,
TErrorInterceptor,
} from './interceptors/interceptor.types.js';
import { InterceptorManager } from './interceptors/interceptor.manager.js';
import { CacheManager } from './cache/cache.manager.js';
import { RetryManager } from './retry/retry.manager.js';
import { RequestDeduplicator } from './utils/deduplicator.js';
import { fetchWithTimeout } from './utils/timeout.js';
export class WebrequestClient {
private interceptorManager: InterceptorManager;
private cacheManager: CacheManager;
private deduplicator: RequestDeduplicator;
private defaultOptions: Partial<IWebrequestOptions>;
constructor(options: Partial<IWebrequestOptions> = {}) {
this.defaultOptions = options;
this.interceptorManager = new InterceptorManager();
this.cacheManager = new CacheManager();
this.deduplicator = new RequestDeduplicator();
}
/**
* Add a global request interceptor
*/
public addRequestInterceptor(interceptor: TRequestInterceptor): void {
this.interceptorManager.addRequestInterceptor(interceptor);
}
/**
* Add a global response interceptor
*/
public addResponseInterceptor(interceptor: TResponseInterceptor): void {
this.interceptorManager.addResponseInterceptor(interceptor);
}
/**
* Add a global error interceptor
*/
public addErrorInterceptor(interceptor: TErrorInterceptor): void {
this.interceptorManager.addErrorInterceptor(interceptor);
}
/**
* Remove a request interceptor
*/
public removeRequestInterceptor(interceptor: TRequestInterceptor): void {
this.interceptorManager.removeRequestInterceptor(interceptor);
}
/**
* Remove a response interceptor
*/
public removeResponseInterceptor(interceptor: TResponseInterceptor): void {
this.interceptorManager.removeResponseInterceptor(interceptor);
}
/**
* Remove an error interceptor
*/
public removeErrorInterceptor(interceptor: TErrorInterceptor): void {
this.interceptorManager.removeErrorInterceptor(interceptor);
}
/**
* Clear all interceptors
*/
public clearInterceptors(): void {
this.interceptorManager.clearAll();
}
/**
* Clear the cache
*/
public async clearCache(): Promise<void> {
await this.cacheManager.clear();
}
/**
* Execute a request with all configured features
*/
public async request(
url: string | Request,
options: IWebrequestOptions = {},
): Promise<Response> {
// Merge default options with request options
const mergedOptions: IWebrequestOptions = {
...this.defaultOptions,
...options,
};
// Create Request object
let request: Request;
if (typeof url === 'string') {
request = new Request(url, mergedOptions);
} else {
request = url;
}
// Process through request interceptors
request = await this.interceptorManager.processRequest(request);
// Add per-request interceptors if provided
if (mergedOptions.interceptors?.request) {
for (const interceptor of mergedOptions.interceptors.request) {
request = await interceptor(request);
}
}
// Execute with deduplication if enabled
const deduplicate = mergedOptions.deduplicate ?? false;
if (deduplicate) {
const dedupeKey = this.deduplicator.generateKey(request);
const result = await this.deduplicator.execute(dedupeKey, async () => {
return await this.executeRequest(request, mergedOptions);
});
return result.response;
}
return await this.executeRequest(request, mergedOptions);
}
/**
* Internal request execution with caching and retry
*/
private async executeRequest(
request: Request,
options: IWebrequestOptions,
): Promise<Response> {
try {
// Determine if retry is enabled
const retryOptions =
typeof options.retry === 'object'
? options.retry
: options.retry
? {}
: undefined;
// Create fetch function for Request objects (used with caching)
const fetchFnForRequest = async (req: Request): Promise<Response> => {
const timeout = options.timeout ?? 60000;
return await fetchWithTimeout(
req.url,
{
method: req.method,
headers: req.headers,
body: req.body,
...options,
},
timeout,
);
};
// Create fetch function for fallbacks (url + init)
const fetchFnForFallbacks = async (url: string, init: RequestInit): Promise<Response> => {
const timeout = options.timeout ?? 60000;
return await fetchWithTimeout(url, init, timeout);
};
let response: Response;
// Execute with retry if enabled
if (retryOptions) {
const retryManager = new RetryManager(retryOptions);
// Handle fallback URLs if provided
if (options.fallbackUrls && options.fallbackUrls.length > 0) {
const allUrls = [request.url, ...options.fallbackUrls];
response = await retryManager.executeWithFallbacks(
allUrls,
{
method: request.method,
headers: request.headers,
body: request.body,
...options,
},
fetchFnForFallbacks,
);
} else {
response = await retryManager.execute(async () => {
// Execute with caching
const result = await this.cacheManager.execute(
request,
options,
fetchFnForRequest,
);
return result.response;
});
}
} else {
// Execute with caching (no retry)
const result = await this.cacheManager.execute(
request,
options,
fetchFnForRequest,
);
response = result.response;
}
// Process through response interceptors
response = await this.interceptorManager.processResponse(response);
// Add per-request response interceptors if provided
if (options.interceptors?.response) {
for (const interceptor of options.interceptors.response) {
response = await interceptor(response);
}
}
return response;
} catch (error) {
// Process through error interceptors
const processedError = await this.interceptorManager.processError(
error instanceof Error ? error : new Error(String(error)),
);
throw processedError;
}
}
/**
* Convenience method: GET request returning JSON
*/
public async getJson<T = any>(
url: string,
options: IWebrequestOptions = {},
): Promise<T> {
const response = await this.request(url, {
...options,
method: 'GET',
headers: {
Accept: 'application/json',
...((options.headers as any) || {}),
},
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await response.json();
}
/**
* Convenience method: POST request with JSON body
*/
public async postJson<T = any>(
url: string,
data: any,
options: IWebrequestOptions = {},
): Promise<T> {
const response = await this.request(url, {
...options,
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
...((options.headers as any) || {}),
},
body: JSON.stringify(data),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await response.json();
}
/**
* Convenience method: PUT request with JSON body
*/
public async putJson<T = any>(
url: string,
data: any,
options: IWebrequestOptions = {},
): Promise<T> {
const response = await this.request(url, {
...options,
method: 'PUT',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
...((options.headers as any) || {}),
},
body: JSON.stringify(data),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await response.json();
}
/**
* Convenience method: DELETE request
*/
public async deleteJson<T = any>(
url: string,
options: IWebrequestOptions = {},
): Promise<T> {
const response = await this.request(url, {
...options,
method: 'DELETE',
headers: {
Accept: 'application/json',
...((options.headers as any) || {}),
},
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await response.json();
}
}

147
ts/webrequest.function.ts Normal file
View File

@@ -0,0 +1,147 @@
/**
* Main webrequest function - fetch-compatible API
*/
import type { IWebrequestOptions } from './webrequest.types.js';
import { WebrequestClient } from './webrequest.client.js';
// Global default client
const defaultClient = new WebrequestClient();
/**
* Fetch-compatible webrequest function
* Drop-in replacement for fetch() with caching, retry, and fault tolerance
*
* @param input - URL or Request object
* @param init - Request options (standard RequestInit + webrequest extensions)
* @returns Promise<Response>
*
* @example
* ```typescript
* // Simple GET request
* const response = await webrequest('https://api.example.com/data');
* const data = await response.json();
*
* // With caching
* const response = await webrequest('https://api.example.com/data', {
* cacheStrategy: 'cache-first',
* cacheMaxAge: 60000
* });
*
* // With retry
* const response = await webrequest('https://api.example.com/data', {
* retry: {
* maxAttempts: 3,
* backoff: 'exponential'
* }
* });
*
* // With fallback URLs
* const response = await webrequest('https://api.example.com/data', {
* fallbackUrls: ['https://backup.example.com/data'],
* retry: true
* });
* ```
*/
export async function webrequest(
input: string | Request | URL,
init?: IWebrequestOptions,
): Promise<Response> {
const url = input instanceof Request ? input.url : String(input);
const request = input instanceof Request ? input : new Request(url, init);
return await defaultClient.request(request, init);
}
/**
* Convenience method: GET request returning JSON
*/
webrequest.getJson = async function <T = any>(
url: string,
options?: IWebrequestOptions,
): Promise<T> {
return await defaultClient.getJson<T>(url, options);
};
/**
* Convenience method: POST request with JSON body
*/
webrequest.postJson = async function <T = any>(
url: string,
data: any,
options?: IWebrequestOptions,
): Promise<T> {
return await defaultClient.postJson<T>(url, data, options);
};
/**
* Convenience method: PUT request with JSON body
*/
webrequest.putJson = async function <T = any>(
url: string,
data: any,
options?: IWebrequestOptions,
): Promise<T> {
return await defaultClient.putJson<T>(url, data, options);
};
/**
* Convenience method: DELETE request
*/
webrequest.deleteJson = async function <T = any>(
url: string,
options?: IWebrequestOptions,
): Promise<T> {
return await defaultClient.deleteJson<T>(url, options);
};
/**
* Add a global request interceptor
*/
webrequest.addRequestInterceptor = function (interceptor) {
defaultClient.addRequestInterceptor(interceptor);
};
/**
* Add a global response interceptor
*/
webrequest.addResponseInterceptor = function (interceptor) {
defaultClient.addResponseInterceptor(interceptor);
};
/**
* Add a global error interceptor
*/
webrequest.addErrorInterceptor = function (interceptor) {
defaultClient.addErrorInterceptor(interceptor);
};
/**
* Clear all global interceptors
*/
webrequest.clearInterceptors = function () {
defaultClient.clearInterceptors();
};
/**
* Clear the cache
*/
webrequest.clearCache = async function () {
await defaultClient.clearCache();
};
/**
* Create a new WebrequestClient with custom configuration
*/
webrequest.createClient = function (
options?: Partial<IWebrequestOptions>,
): WebrequestClient {
return new WebrequestClient(options);
};
/**
* Get the default client
*/
webrequest.getDefaultClient = function (): WebrequestClient {
return defaultClient;
};

143
ts/webrequest.types.ts Normal file
View File

@@ -0,0 +1,143 @@
/**
* Core type definitions for @push.rocks/webrequest v4
*/
// ==================
// Cache Types
// ==================
export type TCacheStrategy =
| 'network-first'
| 'cache-first'
| 'stale-while-revalidate'
| 'network-only'
| 'cache-only';
export type TStandardCacheMode =
| 'default'
| 'no-store'
| 'reload'
| 'no-cache'
| 'force-cache'
| 'only-if-cached';
export interface ICacheEntry {
response: ArrayBuffer;
headers: Record<string, string>;
timestamp: number;
etag?: string;
lastModified?: string;
maxAge?: number;
url: string;
status: number;
statusText: string;
}
export interface ICacheOptions {
/** Standard cache mode (fetch API compatible) */
cache?: TStandardCacheMode;
/** Advanced cache strategy */
cacheStrategy?: TCacheStrategy;
/** Maximum age in milliseconds */
cacheMaxAge?: number;
/** Custom cache key generator */
cacheKey?: string | ((request: Request) => string);
/** Force revalidation even if cached */
revalidate?: boolean;
}
// ==================
// Retry Types
// ==================
export type TBackoffStrategy = 'exponential' | 'linear' | 'constant';
export interface IRetryOptions {
/** Maximum number of retry attempts (default: 3) */
maxAttempts?: number;
/** Backoff strategy (default: 'exponential') */
backoff?: TBackoffStrategy;
/** Initial delay in milliseconds (default: 1000) */
initialDelay?: number;
/** Maximum delay in milliseconds (default: 30000) */
maxDelay?: number;
/** Status codes or function to determine if retry should occur */
retryOn?: number[] | ((response: Response, error?: Error) => boolean);
/** Callback on each retry attempt */
onRetry?: (attempt: number, error: Error, nextDelay: number) => void;
}
// ==================
// Interceptor Types
// ==================
export type TRequestInterceptor = (
request: Request,
) => Request | Promise<Request>;
export type TResponseInterceptor = (
response: Response,
) => Response | Promise<Response>;
export interface IInterceptors {
request?: TRequestInterceptor[];
response?: TResponseInterceptor[];
}
// ==================
// Main Options
// ==================
export interface IWebrequestOptions extends Omit<RequestInit, 'cache'> {
// Caching
cache?: TStandardCacheMode;
cacheStrategy?: TCacheStrategy;
cacheMaxAge?: number;
cacheKey?: string | ((request: Request) => string);
revalidate?: boolean;
// Retry & Fault Tolerance
retry?: boolean | IRetryOptions;
fallbackUrls?: string[];
timeout?: number;
// Interceptors
interceptors?: IInterceptors;
// Deduplication
deduplicate?: boolean;
// Logging
logging?: boolean;
}
// ==================
// Result Types
// ==================
export interface IWebrequestSuccess<T> {
ok: true;
data: T;
response: Response;
}
export interface IWebrequestError {
ok: false;
error: Error;
response?: Response;
}
export type TWebrequestResult<T> = IWebrequestSuccess<T> | IWebrequestError;
// ==================
// Internal Types
// ==================
export interface ICacheMetadata {
maxAge: number;
etag?: string;
lastModified?: string;
immutable: boolean;
noCache: boolean;
noStore: boolean;
mustRevalidate: boolean;
}

View File

@@ -1,14 +1,15 @@
{
"compilerOptions": {
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
"verbatimModuleSyntax": true,
"baseUrl": ".",
"paths": {}
},
"exclude": [
"dist_*/**/*.d.ts"
]
"exclude": ["dist_*/**/*.d.ts"]
}