feat(429 handling): now handles 429 correctly

2025-07-29 13:19:43 +00:00
parent cf24bf94b9
commit 4cbca08f43
7 changed files with 266 additions and 19 deletions
--- a/changelog.md
+++ b/changelog.md
@@ -1,5 +1,18 @@
 # Changelog

+## 2025-07-29 - 4.1.0 - feat(client)
+Add missing options() method to SmartRequest client
+
+**Features:**
+- Added `options()` method to SmartRequest class for setting arbitrary request options
+- Enables setting keepAlive and other platform-specific options via fluent API
+- Added test coverage for keepAlive functionality
+
+**Documentation:**
+- Updated README with examples of using the `options()` method
+- Added specific examples for enabling keepAlive connections
+- Corrected all documentation to use `options()` instead of `option()`
+
 ## 2025-07-28 - 4.0.0 - BREAKING CHANGE(core)
 Complete architectural overhaul with cross-platform support

--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@push.rocks/smartrequest",
-  "version": "4.0.1",
+  "version": "4.1.0",
  "private": false,
  "description": "A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.",
  "exports": {
--- a/readme.md
+++ b/readme.md
@@ -125,6 +125,25 @@ async function fetchWithRetry(url: string) {
 }
 ```

+### Setting Request Options
+
+Use the `options()` method to set any request options supported by the underlying implementation:
+
+```typescript
+import { SmartRequest } from '@push.rocks/smartrequest';
+
+// Set various options
+const response = await SmartRequest.create()
+  .url('https://api.example.com/data')
+  .options({
+    keepAlive: true,           // Enable connection reuse (Node.js)
+    timeout: 10000,            // 10 second timeout
+    hardDataCuttingTimeout: 15000, // 15 second hard timeout
+    // Platform-specific options are also supported
+  })
+  .get();
+```
+
 ### Working with Different Response Types

 The API provides a fetch-like interface for handling different response types:
@@ -326,17 +345,19 @@ import { SmartRequest } from '@push.rocks/smartrequest';

 // Enable keep-alive for better performance with multiple requests
 async function performMultipleRequests() {
-  const client = SmartRequest.create()
-    .header('Connection', 'keep-alive');
+  // Note: keepAlive is NOT enabled by default
+  const response1 = await SmartRequest.create()
+    .url('https://api.example.com/endpoint1')
+    .options({ keepAlive: true })
+    .get();

-  // Requests will reuse the same connection in Node.js
-  const results = await Promise.all([
-    client.url('https://api.example.com/endpoint1').get(),
-    client.url('https://api.example.com/endpoint2').get(),
-    client.url('https://api.example.com/endpoint3').get()
-  ]);
+  const response2 = await SmartRequest.create()
+    .url('https://api.example.com/endpoint2')
+    .options({ keepAlive: true })
+    .get();

-  return Promise.all(results.map(r => r.json()));
+  // Connections are pooled and reused when keepAlive is enabled
+  return [await response1.json(), await response2.json()];
 }
 ```

@@ -349,7 +370,7 @@ When running in a browser, you can use browser-specific fetch options:
 ```typescript
 const response = await SmartRequest.create()
  .url('https://api.example.com/data')
-  .option({
+  .options({
    credentials: 'include', // Include cookies
    mode: 'cors',          // CORS mode
    cache: 'no-cache',     // Cache mode
@@ -367,7 +388,7 @@ import { Agent } from 'https';

 const response = await SmartRequest.create()
  .url('https://api.example.com/data')
-  .option({
+  .options({
    agent: new Agent({ keepAlive: true }), // Custom agent
    socketPath: '/var/run/api.sock',       // Unix socket
  })
--- a/test/test.node.ts
+++ b/test/test.node.ts
@@ -73,7 +73,7 @@ tap.test('client: should handle query parameters', async () => {
 tap.test('client: should handle timeout configuration', async () => {
  // This test just verifies that the timeout method doesn't throw
  const client = SmartRequest.create()
-    .url('https://httpbin.org/get')
+    .url('https://jsonplaceholder.typicode.com/posts/1')
    .timeout(5000);

  const response = await client.get();
@@ -84,7 +84,7 @@ tap.test('client: should handle timeout configuration', async () => {
 tap.test('client: should handle retry configuration', async () => {
  // This test just verifies that the retry method doesn't throw
  const client = SmartRequest.create()
-    .url('https://httpbin.org/get')
+    .url('https://jsonplaceholder.typicode.com/posts/1')
    .retry(1);

  const response = await client.get();
@@ -92,4 +92,111 @@ tap.test('client: should handle retry configuration', async () => {
  expect(response.ok).toBeTrue();
 });

+tap.test('client: should support keepAlive option for connection reuse', async () => {
+  // Test basic keepAlive functionality
+  const responses = [];
+  
+  // Make multiple requests with keepAlive enabled
+  for (let i = 0; i < 3; i++) {
+    const response = await SmartRequest.create()
+      .url('https://jsonplaceholder.typicode.com/posts/1')
+      .options({ keepAlive: true })
+      .header('X-Request-Number', String(i))
+      .get();
+    
+    expect(response.ok).toBeTrue();
+    responses.push(response);
+  }
+  
+  // Verify all requests succeeded
+  expect(responses).toHaveLength(3);
+  
+  // Also test that keepAlive: false works
+  const responseNoKeepAlive = await SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/2')
+    .options({ keepAlive: false })
+    .get();
+    
+  expect(responseNoKeepAlive.ok).toBeTrue();
+  
+  // Verify we can parse the responses
+  const data = await responses[0].json();
+  expect(data).toHaveProperty('id');
+  expect(data.id).toEqual(1);
+});
+
+tap.test('client: should handle 429 rate limiting with default config', async () => {
+  // Test that handle429Backoff can be configured without errors
+  const client = SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/1')
+    .handle429Backoff();
+
+  const response = await client.get();
+  expect(response.status).toEqual(200);
+});
+
+tap.test('client: should handle 429 with custom config', async () => {
+  let rateLimitCallbackCalled = false;
+  let attemptCount = 0;
+  let waitTimeReceived = 0;
+
+  const client = SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/1')
+    .handle429Backoff({
+      maxRetries: 2,
+      fallbackDelay: 500,
+      maxWaitTime: 5000,
+      onRateLimit: (attempt, waitTime) => {
+        rateLimitCallbackCalled = true;
+        attemptCount = attempt;
+        waitTimeReceived = waitTime;
+      }
+    });
+
+  const response = await client.get();
+  expect(response.status).toEqual(200);
+  
+  // The callback should not have been called for a 200 response
+  expect(rateLimitCallbackCalled).toBeFalse();
+});
+
+tap.test('client: should respect Retry-After header format (seconds)', async () => {
+  // Test the configuration works - actual 429 testing would require a mock server
+  const client = SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/1')
+    .handle429Backoff({
+      maxRetries: 1,
+      respectRetryAfter: true
+    });
+
+  const response = await client.get();
+  expect(response.ok).toBeTrue();
+});
+
+tap.test('client: should handle rate limiting with exponential backoff', async () => {
+  // Test exponential backoff configuration
+  const client = SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/1')
+    .handle429Backoff({
+      maxRetries: 3,
+      fallbackDelay: 100,
+      backoffFactor: 2,
+      maxWaitTime: 1000
+    });
+
+  const response = await client.get();
+  expect(response.status).toEqual(200);
+});
+
+tap.test('client: should not retry non-429 errors with rate limit handler', async () => {
+  // Test that 404 errors are not retried by rate limit handler
+  const client = SmartRequest.create()
+    .url('https://jsonplaceholder.typicode.com/posts/999999')
+    .handle429Backoff();
+
+  const response = await client.get();
+  expect(response.status).toEqual(404);
+  expect(response.ok).toBeFalse();
+});
+
 tap.start();
--- a/ts/client/index.ts
+++ b/ts/client/index.ts
@@ -5,7 +5,7 @@ export { SmartRequest } from './smartrequest.js';
 export { CoreResponse } from '../core/index.js';

 // Export types
-export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig } from './types/common.js';
+export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig, RateLimitConfig } from './types/common.js';
 export { 
  PaginationStrategy,
  type TPaginationConfig as PaginationConfig,
--- a/ts/client/smartrequest.ts
+++ b/ts/client/smartrequest.ts
@@ -3,7 +3,7 @@ import type { ICoreResponse } from '../core_base/types.js';
 import * as plugins from './plugins.js';
 import type { ICoreRequestOptions } from '../core_base/types.js';

-import type { HttpMethod, ResponseType, FormField } from './types/common.js';
+import type { HttpMethod, ResponseType, FormField, RateLimitConfig } from './types/common.js';
 import {
  type TPaginationConfig,
  PaginationStrategy,
@@ -14,6 +14,32 @@ import {
 } from './types/pagination.js';
 import { createPaginatedResponse } from './features/pagination.js';

+/**
+ * Parse Retry-After header value to milliseconds
+ * @param retryAfter - The Retry-After header value (seconds or HTTP date)
+ * @returns Delay in milliseconds
+ */
+function parseRetryAfter(retryAfter: string | string[]): number {
+  // Handle array of values (take first)
+  const value = Array.isArray(retryAfter) ? retryAfter[0] : retryAfter;
+  
+  if (!value) return 0;
+  
+  // Try to parse as seconds (number)
+  const seconds = parseInt(value, 10);
+  if (!isNaN(seconds)) {
+    return seconds * 1000;
+  }
+  
+  // Try to parse as HTTP date
+  const retryDate = new Date(value);
+  if (!isNaN(retryDate.getTime())) {
+    return Math.max(0, retryDate.getTime() - Date.now());
+  }
+  
+  return 0;
+}
+
 /**
 * Modern fluent client for making HTTP requests
 */
@@ -23,6 +49,7 @@ export class SmartRequest<T = any> {
  private _retries: number = 0;
  private _queryParams: Record<string, string> = {};
  private _paginationConfig?: TPaginationConfig;
+  private _rateLimitConfig?: RateLimitConfig;

  /**
   * Create a new SmartRequest instance
@@ -106,6 +133,21 @@ export class SmartRequest<T = any> {
    return this;
  }

+  /**
+   * Enable automatic 429 (Too Many Requests) handling with configurable backoff
+   */
+  handle429Backoff(config?: RateLimitConfig): this {
+    this._rateLimitConfig = {
+      maxRetries: config?.maxRetries ?? 3,
+      respectRetryAfter: config?.respectRetryAfter ?? true,
+      maxWaitTime: config?.maxWaitTime ?? 60000,
+      fallbackDelay: config?.fallbackDelay ?? 1000,
+      backoffFactor: config?.backoffFactor ?? 2,
+      onRateLimit: config?.onRateLimit
+    };
+    return this;
+  }
+
  /**
   * Set HTTP headers
   */
@@ -142,6 +184,17 @@ export class SmartRequest<T = any> {
    return this;
  }

+  /**
+   * Set additional request options
+   */
+  options(options: Partial<ICoreRequestOptions>): this {
+    this._options = {
+      ...this._options,
+      ...options
+    };
+    return this;
+  }
+
  /**
   * Set the Accept header to indicate what content type is expected
   */
@@ -305,14 +358,55 @@ export class SmartRequest<T = any> {

    this._options.queryParams = this._queryParams;

-    // Handle retry logic
+    // Track rate limit attempts separately
+    let rateLimitAttempt = 0;
    let lastError: Error;

+    // Main retry loop
    for (let attempt = 0; attempt <= this._retries; attempt++) {
      try {
        const request = new CoreRequest(this._url, this._options as any);
-        const response = await request.fire();
-        return response as ICoreResponse<R>;
+        const response = await request.fire() as ICoreResponse<R>;
+        
+        // Check for 429 status if rate limit handling is enabled
+        if (this._rateLimitConfig && response.status === 429) {
+          if (rateLimitAttempt >= this._rateLimitConfig.maxRetries) {
+            // Max rate limit retries reached, return the 429 response
+            return response;
+          }
+
+          let waitTime: number;
+          
+          if (this._rateLimitConfig.respectRetryAfter && response.headers['retry-after']) {
+            // Parse Retry-After header
+            waitTime = parseRetryAfter(response.headers['retry-after']);
+            
+            // Cap wait time to maxWaitTime
+            waitTime = Math.min(waitTime, this._rateLimitConfig.maxWaitTime);
+          } else {
+            // Use exponential backoff
+            waitTime = Math.min(
+              this._rateLimitConfig.fallbackDelay * Math.pow(this._rateLimitConfig.backoffFactor, rateLimitAttempt),
+              this._rateLimitConfig.maxWaitTime
+            );
+          }
+
+          // Call rate limit callback if provided
+          if (this._rateLimitConfig.onRateLimit) {
+            this._rateLimitConfig.onRateLimit(rateLimitAttempt + 1, waitTime);
+          }
+
+          // Wait before retrying
+          await new Promise(resolve => setTimeout(resolve, waitTime));
+          
+          rateLimitAttempt++;
+          // Decrement attempt to retry this attempt
+          attempt--;
+          continue;
+        }
+        
+        // Success or non-429 error response
+        return response;
      } catch (error) {
        lastError = error as Error;

--- a/ts/client/types/common.ts
+++ b/ts/client/types/common.ts
@@ -46,4 +46,16 @@ export interface TimeoutConfig {
  connection?: number;      // Connection timeout in ms
  socket?: number;          // Socket idle timeout in ms
  response?: number;        // Response timeout in ms
+}
+
+/**
+ * Rate limit configuration for handling 429 responses
+ */
+export interface RateLimitConfig {
+  maxRetries?: number;           // Maximum number of retries (default: 3)
+  respectRetryAfter?: boolean;   // Respect Retry-After header (default: true)
+  maxWaitTime?: number;          // Max wait time in ms (default: 60000)
+  fallbackDelay?: number;        // Delay when no Retry-After header (default: 1000)
+  backoffFactor?: number;        // Exponential backoff factor (default: 2)
+  onRateLimit?: (attempt: number, waitTime: number) => void; // Callback for rate limit events
 }