From 4cbca08f43b33c9a8a4e4211809108c97374df60 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Tue, 29 Jul 2025 13:19:43 +0000 Subject: [PATCH] feat(429 handling): now handles 429 correctly --- changelog.md | 13 +++++ package.json | 2 +- readme.md | 43 +++++++++++---- test/test.node.ts | 111 +++++++++++++++++++++++++++++++++++++- ts/client/index.ts | 2 +- ts/client/smartrequest.ts | 102 +++++++++++++++++++++++++++++++++-- ts/client/types/common.ts | 12 +++++ 7 files changed, 266 insertions(+), 19 deletions(-) diff --git a/changelog.md b/changelog.md index 30b5b8c..596b305 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,18 @@ # Changelog +## 2025-07-29 - 4.1.0 - feat(client) +Add missing options() method to SmartRequest client + +**Features:** +- Added `options()` method to SmartRequest class for setting arbitrary request options +- Enables setting keepAlive and other platform-specific options via fluent API +- Added test coverage for keepAlive functionality + +**Documentation:** +- Updated README with examples of using the `options()` method +- Added specific examples for enabling keepAlive connections +- Corrected all documentation to use `options()` instead of `option()` + ## 2025-07-28 - 4.0.0 - BREAKING CHANGE(core) Complete architectural overhaul with cross-platform support diff --git a/package.json b/package.json index fdc1524..2902def 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@push.rocks/smartrequest", - "version": "4.0.1", + "version": "4.1.0", "private": false, "description": "A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.", "exports": { diff --git a/readme.md b/readme.md index b04aaa6..4987646 100644 --- a/readme.md +++ b/readme.md @@ -125,6 +125,25 @@ async function fetchWithRetry(url: string) { } ``` +### Setting Request Options + +Use the `options()` method to set any request options supported by the underlying implementation: + +```typescript +import { SmartRequest } from '@push.rocks/smartrequest'; + +// Set various options +const response = await SmartRequest.create() + .url('https://api.example.com/data') + .options({ + keepAlive: true, // Enable connection reuse (Node.js) + timeout: 10000, // 10 second timeout + hardDataCuttingTimeout: 15000, // 15 second hard timeout + // Platform-specific options are also supported + }) + .get(); +``` + ### Working with Different Response Types The API provides a fetch-like interface for handling different response types: @@ -326,17 +345,19 @@ import { SmartRequest } from '@push.rocks/smartrequest'; // Enable keep-alive for better performance with multiple requests async function performMultipleRequests() { - const client = SmartRequest.create() - .header('Connection', 'keep-alive'); + // Note: keepAlive is NOT enabled by default + const response1 = await SmartRequest.create() + .url('https://api.example.com/endpoint1') + .options({ keepAlive: true }) + .get(); - // Requests will reuse the same connection in Node.js - const results = await Promise.all([ - client.url('https://api.example.com/endpoint1').get(), - client.url('https://api.example.com/endpoint2').get(), - client.url('https://api.example.com/endpoint3').get() - ]); + const response2 = await SmartRequest.create() + .url('https://api.example.com/endpoint2') + .options({ keepAlive: true }) + .get(); - return Promise.all(results.map(r => r.json())); + // Connections are pooled and reused when keepAlive is enabled + return [await response1.json(), await response2.json()]; } ``` @@ -349,7 +370,7 @@ When running in a browser, you can use browser-specific fetch options: ```typescript const response = await SmartRequest.create() .url('https://api.example.com/data') - .option({ + .options({ credentials: 'include', // Include cookies mode: 'cors', // CORS mode cache: 'no-cache', // Cache mode @@ -367,7 +388,7 @@ import { Agent } from 'https'; const response = await SmartRequest.create() .url('https://api.example.com/data') - .option({ + .options({ agent: new Agent({ keepAlive: true }), // Custom agent socketPath: '/var/run/api.sock', // Unix socket }) diff --git a/test/test.node.ts b/test/test.node.ts index a55f236..8ee7f8a 100644 --- a/test/test.node.ts +++ b/test/test.node.ts @@ -73,7 +73,7 @@ tap.test('client: should handle query parameters', async () => { tap.test('client: should handle timeout configuration', async () => { // This test just verifies that the timeout method doesn't throw const client = SmartRequest.create() - .url('https://httpbin.org/get') + .url('https://jsonplaceholder.typicode.com/posts/1') .timeout(5000); const response = await client.get(); @@ -84,7 +84,7 @@ tap.test('client: should handle timeout configuration', async () => { tap.test('client: should handle retry configuration', async () => { // This test just verifies that the retry method doesn't throw const client = SmartRequest.create() - .url('https://httpbin.org/get') + .url('https://jsonplaceholder.typicode.com/posts/1') .retry(1); const response = await client.get(); @@ -92,4 +92,111 @@ tap.test('client: should handle retry configuration', async () => { expect(response.ok).toBeTrue(); }); +tap.test('client: should support keepAlive option for connection reuse', async () => { + // Test basic keepAlive functionality + const responses = []; + + // Make multiple requests with keepAlive enabled + for (let i = 0; i < 3; i++) { + const response = await SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/1') + .options({ keepAlive: true }) + .header('X-Request-Number', String(i)) + .get(); + + expect(response.ok).toBeTrue(); + responses.push(response); + } + + // Verify all requests succeeded + expect(responses).toHaveLength(3); + + // Also test that keepAlive: false works + const responseNoKeepAlive = await SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/2') + .options({ keepAlive: false }) + .get(); + + expect(responseNoKeepAlive.ok).toBeTrue(); + + // Verify we can parse the responses + const data = await responses[0].json(); + expect(data).toHaveProperty('id'); + expect(data.id).toEqual(1); +}); + +tap.test('client: should handle 429 rate limiting with default config', async () => { + // Test that handle429Backoff can be configured without errors + const client = SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/1') + .handle429Backoff(); + + const response = await client.get(); + expect(response.status).toEqual(200); +}); + +tap.test('client: should handle 429 with custom config', async () => { + let rateLimitCallbackCalled = false; + let attemptCount = 0; + let waitTimeReceived = 0; + + const client = SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/1') + .handle429Backoff({ + maxRetries: 2, + fallbackDelay: 500, + maxWaitTime: 5000, + onRateLimit: (attempt, waitTime) => { + rateLimitCallbackCalled = true; + attemptCount = attempt; + waitTimeReceived = waitTime; + } + }); + + const response = await client.get(); + expect(response.status).toEqual(200); + + // The callback should not have been called for a 200 response + expect(rateLimitCallbackCalled).toBeFalse(); +}); + +tap.test('client: should respect Retry-After header format (seconds)', async () => { + // Test the configuration works - actual 429 testing would require a mock server + const client = SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/1') + .handle429Backoff({ + maxRetries: 1, + respectRetryAfter: true + }); + + const response = await client.get(); + expect(response.ok).toBeTrue(); +}); + +tap.test('client: should handle rate limiting with exponential backoff', async () => { + // Test exponential backoff configuration + const client = SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/1') + .handle429Backoff({ + maxRetries: 3, + fallbackDelay: 100, + backoffFactor: 2, + maxWaitTime: 1000 + }); + + const response = await client.get(); + expect(response.status).toEqual(200); +}); + +tap.test('client: should not retry non-429 errors with rate limit handler', async () => { + // Test that 404 errors are not retried by rate limit handler + const client = SmartRequest.create() + .url('https://jsonplaceholder.typicode.com/posts/999999') + .handle429Backoff(); + + const response = await client.get(); + expect(response.status).toEqual(404); + expect(response.ok).toBeFalse(); +}); + tap.start(); diff --git a/ts/client/index.ts b/ts/client/index.ts index 3f8b1d4..c952b94 100644 --- a/ts/client/index.ts +++ b/ts/client/index.ts @@ -5,7 +5,7 @@ export { SmartRequest } from './smartrequest.js'; export { CoreResponse } from '../core/index.js'; // Export types -export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig } from './types/common.js'; +export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig, RateLimitConfig } from './types/common.js'; export { PaginationStrategy, type TPaginationConfig as PaginationConfig, diff --git a/ts/client/smartrequest.ts b/ts/client/smartrequest.ts index cd109ea..8cf0445 100644 --- a/ts/client/smartrequest.ts +++ b/ts/client/smartrequest.ts @@ -3,7 +3,7 @@ import type { ICoreResponse } from '../core_base/types.js'; import * as plugins from './plugins.js'; import type { ICoreRequestOptions } from '../core_base/types.js'; -import type { HttpMethod, ResponseType, FormField } from './types/common.js'; +import type { HttpMethod, ResponseType, FormField, RateLimitConfig } from './types/common.js'; import { type TPaginationConfig, PaginationStrategy, @@ -14,6 +14,32 @@ import { } from './types/pagination.js'; import { createPaginatedResponse } from './features/pagination.js'; +/** + * Parse Retry-After header value to milliseconds + * @param retryAfter - The Retry-After header value (seconds or HTTP date) + * @returns Delay in milliseconds + */ +function parseRetryAfter(retryAfter: string | string[]): number { + // Handle array of values (take first) + const value = Array.isArray(retryAfter) ? retryAfter[0] : retryAfter; + + if (!value) return 0; + + // Try to parse as seconds (number) + const seconds = parseInt(value, 10); + if (!isNaN(seconds)) { + return seconds * 1000; + } + + // Try to parse as HTTP date + const retryDate = new Date(value); + if (!isNaN(retryDate.getTime())) { + return Math.max(0, retryDate.getTime() - Date.now()); + } + + return 0; +} + /** * Modern fluent client for making HTTP requests */ @@ -23,6 +49,7 @@ export class SmartRequest { private _retries: number = 0; private _queryParams: Record = {}; private _paginationConfig?: TPaginationConfig; + private _rateLimitConfig?: RateLimitConfig; /** * Create a new SmartRequest instance @@ -106,6 +133,21 @@ export class SmartRequest { return this; } + /** + * Enable automatic 429 (Too Many Requests) handling with configurable backoff + */ + handle429Backoff(config?: RateLimitConfig): this { + this._rateLimitConfig = { + maxRetries: config?.maxRetries ?? 3, + respectRetryAfter: config?.respectRetryAfter ?? true, + maxWaitTime: config?.maxWaitTime ?? 60000, + fallbackDelay: config?.fallbackDelay ?? 1000, + backoffFactor: config?.backoffFactor ?? 2, + onRateLimit: config?.onRateLimit + }; + return this; + } + /** * Set HTTP headers */ @@ -142,6 +184,17 @@ export class SmartRequest { return this; } + /** + * Set additional request options + */ + options(options: Partial): this { + this._options = { + ...this._options, + ...options + }; + return this; + } + /** * Set the Accept header to indicate what content type is expected */ @@ -305,14 +358,55 @@ export class SmartRequest { this._options.queryParams = this._queryParams; - // Handle retry logic + // Track rate limit attempts separately + let rateLimitAttempt = 0; let lastError: Error; + // Main retry loop for (let attempt = 0; attempt <= this._retries; attempt++) { try { const request = new CoreRequest(this._url, this._options as any); - const response = await request.fire(); - return response as ICoreResponse; + const response = await request.fire() as ICoreResponse; + + // Check for 429 status if rate limit handling is enabled + if (this._rateLimitConfig && response.status === 429) { + if (rateLimitAttempt >= this._rateLimitConfig.maxRetries) { + // Max rate limit retries reached, return the 429 response + return response; + } + + let waitTime: number; + + if (this._rateLimitConfig.respectRetryAfter && response.headers['retry-after']) { + // Parse Retry-After header + waitTime = parseRetryAfter(response.headers['retry-after']); + + // Cap wait time to maxWaitTime + waitTime = Math.min(waitTime, this._rateLimitConfig.maxWaitTime); + } else { + // Use exponential backoff + waitTime = Math.min( + this._rateLimitConfig.fallbackDelay * Math.pow(this._rateLimitConfig.backoffFactor, rateLimitAttempt), + this._rateLimitConfig.maxWaitTime + ); + } + + // Call rate limit callback if provided + if (this._rateLimitConfig.onRateLimit) { + this._rateLimitConfig.onRateLimit(rateLimitAttempt + 1, waitTime); + } + + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, waitTime)); + + rateLimitAttempt++; + // Decrement attempt to retry this attempt + attempt--; + continue; + } + + // Success or non-429 error response + return response; } catch (error) { lastError = error as Error; diff --git a/ts/client/types/common.ts b/ts/client/types/common.ts index 43f7166..904c989 100644 --- a/ts/client/types/common.ts +++ b/ts/client/types/common.ts @@ -46,4 +46,16 @@ export interface TimeoutConfig { connection?: number; // Connection timeout in ms socket?: number; // Socket idle timeout in ms response?: number; // Response timeout in ms +} + +/** + * Rate limit configuration for handling 429 responses + */ +export interface RateLimitConfig { + maxRetries?: number; // Maximum number of retries (default: 3) + respectRetryAfter?: boolean; // Respect Retry-After header (default: true) + maxWaitTime?: number; // Max wait time in ms (default: 60000) + fallbackDelay?: number; // Delay when no Retry-After header (default: 1000) + backoffFactor?: number; // Exponential backoff factor (default: 2) + onRateLimit?: (attempt: number, waitTime: number) => void; // Callback for rate limit events } \ No newline at end of file