diff --git a/changelog.md b/changelog.md index c9c2b85..94e95e1 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,32 @@ # Changelog +## 2025-07-27 - 3.0.0 - BREAKING CHANGE(core) +Major architectural refactoring with fetch-like API + +**Breaking Changes:** +- Legacy API functions are now imported from `@push.rocks/smartrequest/legacy` instead of the main export +- Modern API response objects now use fetch-like methods (`.json()`, `.text()`, `.arrayBuffer()`, `.stream()`) instead of direct `.body` access +- Renamed `responseType()` method to `accept()` in modern API +- Removed automatic defaults: + - No default keepAlive (must be explicitly set) + - No default timeouts + - No automatic JSON parsing in core +- Complete internal architecture refactoring: + - Core module now always returns raw streams + - Response parsing happens in SmartResponse methods + - Legacy API is now just an adapter over the core module + +**Features:** +- New fetch-like response API with single-use body consumption +- Better TypeScript support and type safety +- Cleaner separation of concerns between request and response +- More predictable behavior aligned with fetch API standards + +**Documentation:** +- Updated all examples to show correct import paths +- Added comprehensive examples for the new response API +- Enhanced migration guide + ## 2025-04-03 - 2.1.0 - feat(docs) Enhance documentation and tests with modern API usage examples and migration guide diff --git a/package.json b/package.json index 227bd16..1ace00c 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,12 @@ { "name": "@push.rocks/smartrequest", - "version": "2.1.0", + "version": "3.0.0", "private": false, "description": "A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.", - "main": "dist_ts/index.js", - "typings": "dist_ts/index.d.ts", + "exports": { + ".": "./dist_ts_web/index.js", + "./legacy": "./dist_ts/legacy/index.js" + }, "type": "module", "scripts": { "test": "(tstest test/ --web)", @@ -29,7 +31,7 @@ "modern web requests", "drop-in replacement" ], - "author": "Lossless GmbH", + "author": "Task Venture Capital GmbH", "license": "MIT", "bugs": { "url": "https://gitlab.com/push.rocks/smartrequest/issues" diff --git a/readme.hints.md b/readme.hints.md index a958352..e76ee14 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1,3 +1,6 @@ +# SmartRequest Architecture Hints + +## Core Features - supports http - supports https - supports unix socks @@ -9,3 +12,43 @@ - continuously updated - uses node native http and https modules - used in modules like @push.rocks/smartproxy and @api.global/typedrequest + +## Architecture Overview (as of latest refactoring) +- The project is now structured with a clean separation between core functionality and API layers +- Core module (ts/core/) contains the essential HTTP request logic using Node.js http/https modules +- **Core always returns raw streams** - no parsing or body collection happens in the core request function +- Modern API (ts/modern/) provides a fluent, chainable interface with fetch-like Response objects +- Legacy API is maintained through a thin adapter layer for backward compatibility + +## Key Components + +### Core Module (ts/core/) +- `request.ts`: Core HTTP/HTTPS request logic with unix socket support and keep-alive agents + - `coreRequest()` always returns a raw Node.js IncomingMessage stream + - No response parsing or body collection happens here +- `response.ts`: SmartResponse class providing fetch-like API + - Methods like `json()`, `text()`, `arrayBuffer()` handle all parsing and body collection + - Response body is streamed and collected only when these methods are called + - Body can only be consumed once (throws error on second attempt) +- `types.ts`: Core TypeScript interfaces and types +- `plugins.ts`: Centralized dependencies + +### Modern API +- SmartRequestClient: Fluent API with method chaining +- Returns SmartResponse objects with fetch-like methods +- Supports pagination, retries, timeouts, and various response types + +### Binary Request Handling +- Binary requests are handled correctly when `responseType: 'binary'` is set +- Response body is kept as Buffer without string conversion +- No automatic transformations applied to binary data + +### Legacy Compatibility +- All legacy functions (getJson, postJson, etc.) are maintained through adapter.ts +- Legacy API returns IExtendedIncomingMessage for backward compatibility +- Modern API can be accessed alongside legacy API + +## Testing +- Use `pnpm test` to run all tests +- Modern API tests use the new SmartResponse methods (response.json(), response.text()) +- Legacy API tests continue to use the body property directly diff --git a/readme.md b/readme.md index 8fc0b6c..59b8321 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,5 @@ # @push.rocks/smartrequest -A module providing a drop-in replacement for the deprecated Request library, focusing on modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, and streams. The library offers both a legacy API and a modern fluent API for maximum flexibility. +A modern HTTP/HTTPS request library for Node.js with support for form data, file uploads, JSON, binary data, streams, and unix sockets. Features both a legacy API for backward compatibility and a modern fetch-like API for new projects. ## Install To install `@push.rocks/smartrequest`, use one of the following commands: @@ -17,22 +17,46 @@ yarn add @push.rocks/smartrequest This will add `@push.rocks/smartrequest` to your project's dependencies. +## Key Features + +- 🚀 **Modern Fetch-like API** - Familiar response methods (`.json()`, `.text()`, `.arrayBuffer()`, `.stream()`) +- 🔄 **Two API Styles** - Legacy function-based API and modern fluent chainable API +- 🌐 **Unix Socket Support** - Connect to local services like Docker +- 📦 **Form Data & File Uploads** - Built-in support for multipart/form-data +- 🔁 **Pagination Support** - Multiple strategies (offset, cursor, Link headers) +- ⚡ **Keep-Alive Connections** - Efficient connection pooling +- 🛡️ **TypeScript First** - Full type safety and IntelliSense support +- 🎯 **Zero Magic Defaults** - Explicit configuration following fetch API principles +- 🔌 **Streaming Support** - Handle large files and real-time data +- 🔧 **Highly Configurable** - Timeouts, retries, headers, and more + ## Usage -`@push.rocks/smartrequest` is designed as a versatile, modern HTTP client library for making HTTP/HTTPS requests. It supports a range of features, including handling form data, file uploads, JSON requests, binary data, streaming, pagination, and much more, all within a modern, promise-based API. + +`@push.rocks/smartrequest` is designed as a versatile, modern HTTP client library for making HTTP/HTTPS requests in Node.js environments. It provides a clean, type-safe API inspired by the native fetch API but with additional features needed for server-side applications. The library provides two distinct APIs: -1. **Legacy API** - Simple function-based API for quick and straightforward HTTP requests -2. **Modern Fluent API** - A chainable, builder-style API for more complex scenarios and better TypeScript integration +1. **Legacy API** - Simple function-based API for quick requests and backward compatibility +2. **Modern Fluent API** - A chainable, fetch-like API for more complex scenarios Below we will cover key usage scenarios of `@push.rocks/smartrequest`, showcasing its capabilities and providing you with a solid starting point to integrate it into your projects. +### Import Guide + +```typescript +// Modern API (recommended for new projects) +import { SmartRequestClient } from '@push.rocks/smartrequest'; + +// Legacy API (for backward compatibility) +import { getJson, postJson, request } from '@push.rocks/smartrequest/legacy'; +``` + ### Simple GET Request For fetching data from a REST API or any web service that returns JSON: ```typescript -import { getJson } from '@push.rocks/smartrequest'; +import { getJson } from '@push.rocks/smartrequest/legacy'; async function fetchGitHubUserInfo(username: string) { const response = await getJson(`https://api.github.com/users/${username}`); @@ -49,7 +73,7 @@ The `getJson` function simplifies the process of sending a GET request and parsi When you need to send JSON data to a server, for example, creating a new resource: ```typescript -import { postJson } from '@push.rocks/smartrequest'; +import { postJson } from '@push.rocks/smartrequest/legacy'; async function createTodoItem(todoDetails: { title: string; completed: boolean }) { const response = await postJson('https://jsonplaceholder.typicode.com/todos', { @@ -68,7 +92,7 @@ createTodoItem({ title: 'Implement smartrequest', completed: false }); `@push.rocks/smartrequest` simplifies the process of uploading files and submitting form data to a server: ```typescript -import { postFormData, IFormField } from '@push.rocks/smartrequest'; +import { postFormData, IFormField } from '@push.rocks/smartrequest/legacy'; async function uploadProfilePicture(formDataFields: IFormField[]) { await postFormData('https://api.example.com/upload', {}, formDataFields); @@ -85,7 +109,7 @@ uploadProfilePicture([ For cases when dealing with large datasets or streaming APIs, `@push.rocks/smartrequest` provides streaming capabilities: ```typescript -import { getStream } from '@push.rocks/smartrequest'; +import { getStream } from '@push.rocks/smartrequest/legacy'; async function streamLargeFile(url: string) { const stream = await getStream(url); @@ -109,7 +133,7 @@ streamLargeFile('https://example.com/largefile'); `@push.rocks/smartrequest` is built to be flexible, allowing you to specify additional options to tailor requests to your needs: ```typescript -import { request, ISmartRequestOptions } from '@push.rocks/smartrequest'; +import { request, ISmartRequestOptions } from '@push.rocks/smartrequest/legacy'; async function customRequestExample() { const options: ISmartRequestOptions = { @@ -131,7 +155,7 @@ customRequestExample(); ## Modern Fluent API -In addition to the legacy API shown above, `@push.rocks/smartrequest` provides a modern, fluent API that offers a more chainable and TypeScript-friendly approach to making HTTP requests. +In addition to the legacy API shown above, `@push.rocks/smartrequest` provides a modern, fluent API with a fetch-like response interface that offers a more chainable and TypeScript-friendly approach to making HTTP requests. ### Basic Usage with the Modern API @@ -144,7 +168,9 @@ async function fetchUserData(userId: number) { .url(`https://jsonplaceholder.typicode.com/users/${userId}`) .get(); - console.log(response.body); // The JSON response + // Use the fetch-like response API + const userData = await response.json(); + console.log(userData); // The parsed JSON response } // POST request with JSON body @@ -154,7 +180,8 @@ async function createPost(title: string, body: string, userId: number) { .json({ title, body, userId }) .post(); - console.log(response.body); // The created post + const createdPost = await response.json(); + console.log(createdPost); // The created post } ``` @@ -173,7 +200,8 @@ async function searchRepositories(query: string, perPage: number = 10) { }) .get(); - return response.body.items; + const data = await response.json(); + return data.items; } ``` @@ -189,41 +217,62 @@ async function fetchWithRetry(url: string) { .retry(3) // Retry up to 3 times on failure .get(); - return response.body; + return await response.json(); } ``` ### Working with Different Response Types +The modern API provides a fetch-like interface for handling different response types: + ```typescript import { SmartRequestClient } from '@push.rocks/smartrequest'; +// JSON response (default) +async function fetchJson(url: string) { + const response = await SmartRequestClient.create() + .url(url) + .get(); + + return await response.json(); // Parses JSON automatically +} + +// Text response +async function fetchText(url: string) { + const response = await SmartRequestClient.create() + .url(url) + .get(); + + return await response.text(); // Returns response as string +} + // Binary data async function downloadImage(url: string) { const response = await SmartRequestClient.create() .url(url) - .responseType('binary') + .accept('binary') // Optional: hints to server we want binary .get(); - // response.body is a Buffer - return response.body; + const buffer = await response.arrayBuffer(); + return Buffer.from(buffer); // Convert ArrayBuffer to Buffer if needed } // Streaming response async function streamLargeFile(url: string) { const response = await SmartRequestClient.create() .url(url) - .responseType('stream') .get(); - // response is a stream - response.on('data', (chunk) => { + // Get the underlying Node.js stream + const stream = response.stream(); + + stream.on('data', (chunk) => { console.log(`Received ${chunk.length} bytes of data`); }); return new Promise((resolve, reject) => { - response.on('end', resolve); - response.on('error', reject); + stream.on('end', resolve); + stream.on('error', reject); }); } ``` @@ -289,29 +338,78 @@ async function fetchAllIssues(repo: string) { } ``` -### Convenience Factory Functions +### Advanced Features -The library provides several factory functions for common use cases: +#### Unix Socket Support ```typescript -import { createJsonClient, createBinaryClient, createStreamClient } from '@push.rocks/smartrequest'; +import { SmartRequestClient } from '@push.rocks/smartrequest'; -// Pre-configured for JSON requests -const jsonClient = createJsonClient() - .url('https://api.example.com/data') - .get(); - -// Pre-configured for binary data -const binaryClient = createBinaryClient() - .url('https://example.com/image.jpg') - .get(); - -// Pre-configured for streaming -const streamClient = createStreamClient() - .url('https://example.com/large-file') - .get(); +// Connect to a service via Unix socket +async function queryViaUnixSocket() { + const response = await SmartRequestClient.create() + .url('http://unix:/var/run/docker.sock:/v1.24/containers/json') + .get(); + + return await response.json(); +} ``` +#### Form Data with File Uploads + +```typescript +import { SmartRequestClient } from '@push.rocks/smartrequest'; + +async function uploadMultipleFiles(files: Array<{name: string, path: string}>) { + const formFields = files.map(file => ({ + name: 'files', + value: fs.readFileSync(file.path), + filename: file.name, + contentType: 'application/octet-stream' + })); + + const response = await SmartRequestClient.create() + .url('https://api.example.com/upload') + .formData(formFields) + .post(); + + return await response.json(); +} +``` + +#### Keep-Alive Connections + +```typescript +import { SmartRequestClient } from '@push.rocks/smartrequest'; + +// Enable keep-alive for better performance with multiple requests +async function performMultipleRequests() { + const client = SmartRequestClient.create() + .header('Connection', 'keep-alive'); + + // Requests will reuse the same connection + const results = await Promise.all([ + client.url('https://api.example.com/endpoint1').get(), + client.url('https://api.example.com/endpoint2').get(), + client.url('https://api.example.com/endpoint3').get() + ]); + + return Promise.all(results.map(r => r.json())); +} +``` + +### Response Object Methods + +The modern API returns a `SmartResponse` object with the following methods: + +- `json(): Promise` - Parse response as JSON +- `text(): Promise` - Get response as text +- `arrayBuffer(): Promise` - Get response as ArrayBuffer +- `stream(): NodeJS.ReadableStream` - Get the underlying Node.js stream +- `raw(): http.IncomingMessage` - Get the raw http.IncomingMessage + +Each body method can only be called once per response, similar to the fetch API. + Through its comprehensive set of features tailored for modern web development, `@push.rocks/smartrequest` aims to provide developers with a powerful tool for handling HTTP/HTTPS requests efficiently. Whether it's a simple API call, handling form data, processing streams, or working with paginated APIs, `@push.rocks/smartrequest` delivers a robust, type-safe solution to fit your project's requirements. ## Migration Guide: Legacy API to Modern API @@ -325,11 +423,121 @@ If you're currently using the legacy API and want to migrate to the modern fluen | `putJson(url, { requestBody: data })` | `SmartRequestClient.create().url(url).json(data).put()` | | `delJson(url)` | `SmartRequestClient.create().url(url).delete()` | | `postFormData(url, {}, fields)` | `SmartRequestClient.create().url(url).formData(fields).post()` | -| `getStream(url)` | `SmartRequestClient.create().url(url).responseType('stream').get()` | +| `getStream(url)` | `SmartRequestClient.create().url(url).accept('stream').get()` | | `request(url, options)` | `SmartRequestClient.create().url(url).[...configure options...].get()` | The modern API provides more flexibility and better TypeScript integration, making it the recommended approach for new projects. +## Complete Examples + +### Building a REST API Client + +Here's a complete example of building a typed API client using smartrequest: + +```typescript +import { SmartRequestClient, type SmartResponse } from '@push.rocks/smartrequest'; + +interface User { + id: number; + name: string; + email: string; +} + +interface Post { + id: number; + title: string; + body: string; + userId: number; +} + +class BlogApiClient { + private baseUrl = 'https://jsonplaceholder.typicode.com'; + + private async request(path: string) { + return SmartRequestClient.create() + .url(`${this.baseUrl}${path}`) + .header('Accept', 'application/json'); + } + + async getUser(id: number): Promise { + const response = await this.request(`/users/${id}`).get(); + return response.json(); + } + + async createPost(post: Omit): Promise { + const response = await this.request('/posts') + .json(post) + .post(); + return response.json(); + } + + async deletePost(id: number): Promise { + const response = await this.request(`/posts/${id}`).delete(); + + if (!response.ok) { + throw new Error(`Failed to delete post: ${response.statusText}`); + } + } + + async getAllPosts(userId?: number): Promise { + const client = this.request('/posts'); + + if (userId) { + client.query({ userId: userId.toString() }); + } + + const response = await client.get(); + return response.json(); + } +} + +// Usage +const api = new BlogApiClient(); +const user = await api.getUser(1); +const posts = await api.getAllPosts(user.id); +``` + +### Error Handling + +```typescript +import { SmartRequestClient } from '@push.rocks/smartrequest'; + +async function fetchWithErrorHandling(url: string) { + try { + const response = await SmartRequestClient.create() + .url(url) + .timeout(5000) + .retry(2) + .get(); + + // Check if request was successful + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + // Handle different content types + const contentType = response.headers['content-type']; + + if (contentType?.includes('application/json')) { + return await response.json(); + } else if (contentType?.includes('text/')) { + return await response.text(); + } else { + return await response.arrayBuffer(); + } + } catch (error) { + if (error.code === 'ECONNREFUSED') { + console.error('Connection refused - is the server running?'); + } else if (error.code === 'ETIMEDOUT') { + console.error('Request timed out'); + } else { + console.error('Request failed:', error.message); + } + throw error; + } +} +``` + ## License and Legal Information This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. diff --git a/readme.plan.md b/readme.plan.md new file mode 100644 index 0000000..4f0b42a --- /dev/null +++ b/readme.plan.md @@ -0,0 +1,53 @@ +# Smartrequest Refactoring Plan + +Command to reread CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md` + +## Objective +Refactor smartrequest to use native fetch-like API with a streamlined core that supports unix sockets and keep-alive. + +## Architecture Overview +- Rename `legacy/` to `core/` and remove "smartrequest." prefix from filenames +- Create a modern Response class similar to fetch API +- Use core as foundation for modern API, not as legacy adapter +- Maintain unix socket and keep-alive support + +## Task Checklist + +- [x] Reread /home/philkunz/.claude/CLAUDE.md +- [x] Create ts/core directory structure with request.ts, types.ts, and plugins.ts +- [x] Migrate core request logic from legacy to core/request.ts +- [x] Create modern Response class with fetch-like API +- [x] Update modern API to use new core module +- [x] Create legacy adapter for backward compatibility +- [x] Update exports in ts/index.ts +- [x] Run tests and fix any issues +- [x] Clean up old legacy files + +## Implementation Details + +### Core Module Structure +``` +ts/core/ +├── request.ts # Core HTTP/HTTPS request logic with unix socket support +├── types.ts # Core interfaces and types +├── plugins.ts # Dependencies (http, https, agentkeepalive, etc.) +└── response.ts # Modern Response class +``` + +### Response Class API +The new Response class will provide fetch-like methods: +- `json()`: Promise - Parse response as JSON +- `text()`: Promise - Get response as text +- `arrayBuffer()`: Promise - Get response as ArrayBuffer +- `stream()`: ReadableStream - Get response as stream +- `ok`: boolean - Status is 2xx +- `status`: number - HTTP status code +- `statusText`: string - HTTP status text +- `headers`: Headers - Response headers + +### Migration Strategy +1. Move core request logic without breaking changes +2. Create Response wrapper that provides modern API +3. Update SmartRequestClient to use new core +4. Add legacy adapter for backward compatibility +5. Ensure all tests pass throughout migration \ No newline at end of file diff --git a/test/test.modern.ts b/test/test.modern.ts index 386e670..e929a13 100644 --- a/test/test.modern.ts +++ b/test/test.modern.ts @@ -7,7 +7,11 @@ tap.test('modern: should request a html document over https', async () => { .url('https://encrypted.google.com/') .get(); - expect(response).toHaveProperty('body'); + expect(response).not.toBeNull(); + expect(response).toHaveProperty('status'); + expect(response.status).toBeGreaterThan(0); + const text = await response.text(); + expect(text.length).toBeGreaterThan(0); }); tap.test('modern: should request a JSON document over https', async () => { @@ -15,8 +19,9 @@ tap.test('modern: should request a JSON document over https', async () => { .url('https://jsonplaceholder.typicode.com/posts/1') .get(); - expect(response.body).toHaveProperty('id'); - expect(response.body.id).toEqual(1); + const body = await response.json(); + expect(body).toHaveProperty('id'); + expect(body.id).toEqual(1); }); tap.test('modern: should post a JSON document over http', async () => { @@ -26,9 +31,10 @@ tap.test('modern: should post a JSON document over http', async () => { .json(testData) .post(); - expect(response.body).toHaveProperty('json'); - expect(response.body.json).toHaveProperty('text'); - expect(response.body.json.text).toEqual('example_text'); + const body = await response.json(); + expect(body).toHaveProperty('json'); + expect(body.json).toHaveProperty('text'); + expect(body.json.text).toEqual('example_text'); }); tap.test('modern: should set headers correctly', async () => { @@ -40,12 +46,12 @@ tap.test('modern: should set headers correctly', async () => { .header(customHeader, headerValue) .get(); - - expect(response.body).toHaveProperty('headers'); + const body = await response.json(); + expect(body).toHaveProperty('headers'); // Check if the header exists (case-sensitive) - expect(response.body.headers).toHaveProperty(customHeader); - expect(response.body.headers[customHeader]).toEqual(headerValue); + expect(body.headers).toHaveProperty(customHeader); + expect(body.headers[customHeader]).toEqual(headerValue); }); tap.test('modern: should handle query parameters', async () => { @@ -56,11 +62,12 @@ tap.test('modern: should handle query parameters', async () => { .query(params) .get(); - expect(response.body).toHaveProperty('args'); - expect(response.body.args).toHaveProperty('param1'); - expect(response.body.args.param1).toEqual('value1'); - expect(response.body.args).toHaveProperty('param2'); - expect(response.body.args.param2).toEqual('value2'); + const body = await response.json(); + expect(body).toHaveProperty('args'); + expect(body.args).toHaveProperty('param1'); + expect(body.args.param1).toEqual('value1'); + expect(body.args).toHaveProperty('param2'); + expect(body.args.param2).toEqual('value2'); }); tap.test('modern: should handle timeout configuration', async () => { @@ -70,7 +77,8 @@ tap.test('modern: should handle timeout configuration', async () => { .timeout(5000); const response = await client.get(); - expect(response).toHaveProperty('body'); + expect(response).toHaveProperty('ok'); + expect(response.ok).toBeTrue(); }); tap.test('modern: should handle retry configuration', async () => { @@ -80,7 +88,8 @@ tap.test('modern: should handle retry configuration', async () => { .retry(1); const response = await client.get(); - expect(response).toHaveProperty('body'); + expect(response).toHaveProperty('ok'); + expect(response.ok).toBeTrue(); }); tap.start(); diff --git a/ts/core/index.ts b/ts/core/index.ts new file mode 100644 index 0000000..d4369e4 --- /dev/null +++ b/ts/core/index.ts @@ -0,0 +1,4 @@ +// Core exports +export * from './types.js'; +export * from './response.js'; +export { request, coreRequest, isUnixSocket, parseUnixSocketUrl } from './request.js'; \ No newline at end of file diff --git a/ts/legacy/smartrequest.plugins.ts b/ts/core/plugins.ts similarity index 92% rename from ts/legacy/smartrequest.plugins.ts rename to ts/core/plugins.ts index 9f33a01..2b35a5d 100644 --- a/ts/legacy/smartrequest.plugins.ts +++ b/ts/core/plugins.ts @@ -16,4 +16,4 @@ export { smartpromise, smarturl }; import agentkeepalive from 'agentkeepalive'; import formData from 'form-data'; -export { agentkeepalive, formData }; +export { agentkeepalive, formData }; \ No newline at end of file diff --git a/ts/core/request.ts b/ts/core/request.ts new file mode 100644 index 0000000..b3723e3 --- /dev/null +++ b/ts/core/request.ts @@ -0,0 +1,159 @@ +import * as plugins from './plugins.js'; +import * as types from './types.js'; +import { SmartResponse } from './response.js'; + +// Keep-alive agents for connection pooling +const httpAgent = new plugins.agentkeepalive({ + keepAlive: true, + maxFreeSockets: 10, + maxSockets: 100, + maxTotalSockets: 1000, +}); + +const httpAgentKeepAliveFalse = new plugins.agentkeepalive({ + keepAlive: false, +}); + +const httpsAgent = new plugins.agentkeepalive.HttpsAgent({ + keepAlive: true, + maxFreeSockets: 10, + maxSockets: 100, + maxTotalSockets: 1000, +}); + +const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({ + keepAlive: false, +}); + +/** + * Tests if a URL is a unix socket + */ +export const isUnixSocket = (url: string): boolean => { + const unixRegex = /^(http:\/\/|https:\/\/|)unix:/; + return unixRegex.test(url); +}; + +/** + * Parses socket path and route from unix socket URL + */ +export const parseUnixSocketUrl = (url: string): { socketPath: string; path: string } => { + const parseRegex = /(.*):(.*)/; + const result = parseRegex.exec(url); + return { + socketPath: result[1], + path: result[2], + }; +}; + + +/** + * Core request function that handles all HTTP/HTTPS requests + */ +export async function coreRequest( + urlArg: string, + optionsArg: types.ICoreRequestOptions = {}, + requestDataFunc: ((req: plugins.http.ClientRequest) => void) | null = null +): Promise { + const done = plugins.smartpromise.defer(); + + // No defaults - let users explicitly set options to match fetch behavior + + // Parse URL + const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(urlArg, { + searchParams: optionsArg.queryParams || {}, + }); + + optionsArg.hostname = parsedUrl.hostname; + if (parsedUrl.port) { + optionsArg.port = parseInt(parsedUrl.port, 10); + } + optionsArg.path = parsedUrl.path; + + // Handle unix socket URLs + if (isUnixSocket(urlArg)) { + const { socketPath, path } = parseUnixSocketUrl(optionsArg.path); + optionsArg.socketPath = socketPath; + optionsArg.path = path; + } + + // Determine agent based on protocol and keep-alive setting + if (!optionsArg.agent) { + // Only use keep-alive agents if explicitly requested + if (optionsArg.keepAlive === true) { + optionsArg.agent = parsedUrl.protocol === 'https:' ? httpsAgent : httpAgent; + } else if (optionsArg.keepAlive === false) { + optionsArg.agent = parsedUrl.protocol === 'https:' ? httpsAgentKeepAliveFalse : httpAgentKeepAliveFalse; + } + // If keepAlive is undefined, don't set any agent (more fetch-like behavior) + } + + // Determine request module + const requestModule = parsedUrl.protocol === 'https:' ? plugins.https : plugins.http; + + if (!requestModule) { + throw new Error(`The request to ${urlArg} is missing a viable protocol. Must be http or https`); + } + + // Perform the request + const request = requestModule.request(optionsArg, async (response) => { + // Handle hard timeout + if (optionsArg.hardDataCuttingTimeout) { + setTimeout(() => { + response.destroy(); + done.reject(new Error('Request timed out')); + }, optionsArg.hardDataCuttingTimeout); + } + + // Always return the raw stream + done.resolve(response); + }); + + // Write request body + if (optionsArg.requestBody) { + if (optionsArg.requestBody instanceof plugins.formData) { + optionsArg.requestBody.pipe(request).on('finish', () => { + request.end(); + }); + } else { + // Write body as-is - caller is responsible for serialization + const bodyData = typeof optionsArg.requestBody === 'string' + ? optionsArg.requestBody + : optionsArg.requestBody instanceof Buffer + ? optionsArg.requestBody + : JSON.stringify(optionsArg.requestBody); // Still stringify for backward compatibility + request.write(bodyData); + request.end(); + } + } else if (requestDataFunc) { + requestDataFunc(request); + } else { + request.end(); + } + + // Handle request errors + request.on('error', (e) => { + console.error(e); + request.destroy(); + done.reject(e); + }); + + // Get response and handle response errors + const response = await done.promise; + response.on('error', (err) => { + console.error(err); + response.destroy(); + }); + + return response; +} + +/** + * Modern request function that returns a SmartResponse + */ +export async function request( + urlArg: string, + optionsArg: types.ICoreRequestOptions = {} +): Promise { + const response = await coreRequest(urlArg, optionsArg); + return new SmartResponse(response, urlArg); +} \ No newline at end of file diff --git a/ts/core/response.ts b/ts/core/response.ts new file mode 100644 index 0000000..032d3c8 --- /dev/null +++ b/ts/core/response.ts @@ -0,0 +1,110 @@ +import * as plugins from './plugins.js'; +import * as types from './types.js'; + +/** + * Modern Response class that provides a fetch-like API + */ +export class SmartResponse implements types.ICoreResponse { + private incomingMessage: plugins.http.IncomingMessage; + private bodyBufferPromise: Promise | null = null; + private consumed = false; + + // Public properties + public readonly ok: boolean; + public readonly status: number; + public readonly statusText: string; + public readonly headers: plugins.http.IncomingHttpHeaders; + public readonly url: string; + + constructor(incomingMessage: plugins.http.IncomingMessage, url: string) { + this.incomingMessage = incomingMessage; + this.url = url; + this.status = incomingMessage.statusCode || 0; + this.statusText = incomingMessage.statusMessage || ''; + this.ok = this.status >= 200 && this.status < 300; + this.headers = incomingMessage.headers; + } + + /** + * Ensures the body can only be consumed once + */ + private ensureNotConsumed(): void { + if (this.consumed) { + throw new Error('Body has already been consumed'); + } + this.consumed = true; + } + + /** + * Collects the body as a buffer + */ + private async collectBody(): Promise { + this.ensureNotConsumed(); + + if (this.bodyBufferPromise) { + return this.bodyBufferPromise; + } + + this.bodyBufferPromise = new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + + this.incomingMessage.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + + this.incomingMessage.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + + this.incomingMessage.on('error', reject); + }); + + return this.bodyBufferPromise; + } + + /** + * Parse response as JSON + */ + async json(): Promise { + const buffer = await this.collectBody(); + const text = buffer.toString('utf-8'); + + try { + return JSON.parse(text); + } catch (error) { + throw new Error(`Failed to parse JSON: ${error.message}`); + } + } + + /** + * Get response as text + */ + async text(): Promise { + const buffer = await this.collectBody(); + return buffer.toString('utf-8'); + } + + /** + * Get response as ArrayBuffer + */ + async arrayBuffer(): Promise { + const buffer = await this.collectBody(); + return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength); + } + + /** + * Get response as a readable stream + */ + stream(): NodeJS.ReadableStream { + this.ensureNotConsumed(); + return this.incomingMessage; + } + + /** + * Get the raw IncomingMessage (for legacy compatibility) + */ + raw(): plugins.http.IncomingMessage { + return this.incomingMessage; + } + +} \ No newline at end of file diff --git a/ts/core/types.ts b/ts/core/types.ts new file mode 100644 index 0000000..539ef7d --- /dev/null +++ b/ts/core/types.ts @@ -0,0 +1,67 @@ +import * as plugins from './plugins.js'; + +/** + * Core request options extending Node.js RequestOptions + */ +export interface ICoreRequestOptions extends plugins.https.RequestOptions { + keepAlive?: boolean; + requestBody?: any; + queryParams?: { [key: string]: string }; + hardDataCuttingTimeout?: number; +} + +/** + * HTTP Methods supported + */ +export type THttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS'; + +/** + * Response types supported + */ +export type ResponseType = 'json' | 'text' | 'binary' | 'stream'; + +/** + * Extended IncomingMessage with body property (legacy compatibility) + */ +export interface IExtendedIncomingMessage extends plugins.http.IncomingMessage { + body: T; +} + +/** + * Form field data for multipart/form-data requests + */ +export interface IFormField { + name: string; + value: string | Buffer; + filename?: string; + contentType?: string; +} + +/** + * URL encoded form field + */ +export interface IUrlEncodedField { + key: string; + value: string; +} + +/** + * Core response object that provides fetch-like API + */ +export interface ICoreResponse { + // Properties + ok: boolean; + status: number; + statusText: string; + headers: plugins.http.IncomingHttpHeaders; + url: string; + + // Methods + json(): Promise; + text(): Promise; + arrayBuffer(): Promise; + stream(): NodeJS.ReadableStream; + + // Legacy compatibility + raw(): plugins.http.IncomingMessage; +} \ No newline at end of file diff --git a/ts/index.ts b/ts/index.ts index 2bcf917..7e61bff 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -1,16 +1,12 @@ // Legacy API exports (for backward compatibility) -export { request, safeGet } from './legacy/smartrequest.request.js'; -export type { IExtendedIncomingMessage } from './legacy/smartrequest.request.js'; -export type { ISmartRequestOptions } from './legacy/smartrequest.interfaces.js'; - -export * from './legacy/smartrequest.jsonrest.js'; -export * from './legacy/smartrequest.binaryrest.js'; -export * from './legacy/smartrequest.formdata.js'; -export * from './legacy/smartrequest.stream.js'; +export * from './legacy/index.js'; // Modern API exports export * from './modern/index.js'; -import { SmartRequestClient } from './modern/smartrequestclient.js'; + +// Core exports for advanced usage +export { SmartResponse, type ICoreRequestOptions, type ICoreResponse } from './core/index.js'; // Default export for easier importing +import { SmartRequestClient } from './modern/smartrequestclient.js'; export default SmartRequestClient; \ No newline at end of file diff --git a/ts/legacy/adapter.ts b/ts/legacy/adapter.ts new file mode 100644 index 0000000..0f8019b --- /dev/null +++ b/ts/legacy/adapter.ts @@ -0,0 +1,242 @@ +/** + * Legacy adapter that provides backward compatibility + * Maps legacy API to the new core module + */ + +import * as core from '../core/index.js'; +import * as plugins from '../core/plugins.js'; + +const smartpromise = plugins.smartpromise; + +// Re-export types for backward compatibility +export { type IExtendedIncomingMessage } from '../core/types.js'; +export interface ISmartRequestOptions extends core.ICoreRequestOptions { + autoJsonParse?: boolean; + responseType?: 'json' | 'text' | 'binary' | 'stream'; +} + +// Re-export interface for form fields +export interface IFormField { + name: string; + type: 'string' | 'filePath' | 'Buffer'; + payload: string | Buffer; + fileName?: string; + contentType?: string; +} + +/** + * Helper function to convert stream to IExtendedIncomingMessage for legacy compatibility + */ +async function streamToExtendedMessage( + stream: plugins.http.IncomingMessage, + autoJsonParse = true +): Promise { + const done = smartpromise.defer(); + const chunks: Buffer[] = []; + + stream.on('data', (chunk: Buffer) => { + chunks.push(chunk); + }); + + stream.on('end', () => { + const buffer = Buffer.concat(chunks); + const extendedMessage = stream as core.IExtendedIncomingMessage; + + if (autoJsonParse) { + const text = buffer.toString('utf-8'); + try { + extendedMessage.body = JSON.parse(text); + } catch (err) { + extendedMessage.body = text; + } + } else { + extendedMessage.body = buffer; + } + + done.resolve(extendedMessage); + }); + + stream.on('error', (err) => { + done.reject(err); + }); + + return done.promise; +} + +/** + * Legacy request function that returns IExtendedIncomingMessage + */ +export async function request( + urlArg: string, + optionsArg: ISmartRequestOptions = {}, + responseStreamArg = false, + requestDataFunc?: (req: plugins.http.ClientRequest) => void +): Promise { + const stream = await core.coreRequest(urlArg, optionsArg, requestDataFunc); + + if (responseStreamArg) { + // For stream responses, just cast and return + return stream as core.IExtendedIncomingMessage; + } + + // Convert stream to IExtendedIncomingMessage + const autoJsonParse = optionsArg.autoJsonParse !== false; + return streamToExtendedMessage(stream, autoJsonParse); +} + +/** + * Safe GET request + */ +export async function safeGet(urlArg: string): Promise { + const agentToUse = urlArg.startsWith('http://') + ? new plugins.http.Agent() + : new plugins.https.Agent(); + + try { + const response = await request(urlArg, { + method: 'GET', + agent: agentToUse, + timeout: 5000, + hardDataCuttingTimeout: 5000, + autoJsonParse: false, + }); + return response; + } catch (err) { + console.error(err); + return null; + } +} + +/** + * GET JSON request + */ +export async function getJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) { + optionsArg.method = 'GET'; + return request(urlArg, optionsArg); +} + +/** + * POST JSON request + */ +export async function postJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) { + optionsArg.method = 'POST'; + if ( + typeof optionsArg.requestBody === 'object' && + (!optionsArg.headers || !optionsArg.headers['Content-Type']) + ) { + // make sure headers exist + if (!optionsArg.headers) { + optionsArg.headers = {}; + } + + // assign the right Content-Type, leaving all other headers in place + optionsArg.headers = { + ...optionsArg.headers, + 'Content-Type': 'application/json', + }; + } + return request(urlArg, optionsArg); +} + +/** + * PUT JSON request + */ +export async function putJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) { + optionsArg.method = 'PUT'; + return request(urlArg, optionsArg); +} + +/** + * DELETE JSON request + */ +export async function delJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) { + optionsArg.method = 'DELETE'; + return request(urlArg, optionsArg); +} + +/** + * GET binary data + */ +export async function getBinary(urlArg: string, optionsArg: ISmartRequestOptions = {}) { + optionsArg = { + ...optionsArg, + autoJsonParse: false, + responseType: 'binary' + }; + return request(urlArg, optionsArg); +} + +/** + * POST form data + */ +export async function postFormData(urlArg: string, formFields: IFormField[], optionsArg: ISmartRequestOptions = {}) { + const form = new plugins.formData(); + + for (const formField of formFields) { + if (formField.type === 'filePath') { + const fileData = plugins.fs.readFileSync( + plugins.path.isAbsolute(formField.payload as string) + ? formField.payload as string + : plugins.path.join(process.cwd(), formField.payload as string) + ); + form.append(formField.name, fileData, { + filename: formField.fileName || plugins.path.basename(formField.payload as string), + contentType: formField.contentType + }); + } else if (formField.type === 'Buffer') { + form.append(formField.name, formField.payload, { + filename: formField.fileName, + contentType: formField.contentType + }); + } else { + form.append(formField.name, formField.payload); + } + } + + optionsArg.method = 'POST'; + optionsArg.requestBody = form; + if (!optionsArg.headers) { + optionsArg.headers = {}; + } + optionsArg.headers = { + ...optionsArg.headers, + ...form.getHeaders() + }; + + return request(urlArg, optionsArg); +} + +/** + * POST URL encoded form data + */ +export async function postFormDataUrlEncoded( + urlArg: string, + formFields: { key: string; content: string }[], + optionsArg: ISmartRequestOptions = {} +) { + optionsArg.method = 'POST'; + if (!optionsArg.headers) { + optionsArg.headers = {}; + } + optionsArg.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + + const urlEncodedBody = formFields + .map(field => `${encodeURIComponent(field.key)}=${encodeURIComponent(field.content)}`) + .join('&'); + + optionsArg.requestBody = urlEncodedBody; + + return request(urlArg, optionsArg); +} + +/** + * GET stream + */ +export async function getStream( + urlArg: string, + optionsArg: ISmartRequestOptions = {} +): Promise { + optionsArg.method = 'GET'; + const response = await request(urlArg, optionsArg, true); + return response; +} \ No newline at end of file diff --git a/ts/legacy/index.ts b/ts/legacy/index.ts index 015e3cb..cee4504 100644 --- a/ts/legacy/index.ts +++ b/ts/legacy/index.ts @@ -1,8 +1,2 @@ -export { request, safeGet } from './smartrequest.request.js'; -export type { IExtendedIncomingMessage } from './smartrequest.request.js'; -export type { ISmartRequestOptions } from './smartrequest.interfaces.js'; - -export * from './smartrequest.jsonrest.js'; -export * from './smartrequest.binaryrest.js'; -export * from './smartrequest.formdata.js'; -export * from './smartrequest.stream.js'; +// Export everything from the legacy adapter +export * from './adapter.js'; diff --git a/ts/legacy/smartrequest.binaryrest.ts b/ts/legacy/smartrequest.binaryrest.ts deleted file mode 100644 index a790c30..0000000 --- a/ts/legacy/smartrequest.binaryrest.ts +++ /dev/null @@ -1,33 +0,0 @@ -// this file implements methods to get and post binary data. -import * as interfaces from './smartrequest.interfaces.js'; -import { request, type IExtendedIncomingMessage } from './smartrequest.request.js'; - -import * as plugins from './smartrequest.plugins.js'; - -export const getBinary = async ( - domainArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -) => { - optionsArg = { - ...optionsArg, - autoJsonParse: false, - }; - const done = plugins.smartpromise.defer(); - const response = await request(domainArg, optionsArg, true); - const data: Array = []; - - response - .on('data', function (chunk: Buffer) { - data.push(chunk); - }) - .on('end', function () { - //at this point data is an array of Buffers - //so Buffer.concat() can make us a new Buffer - //of all of them together - const buffer = Buffer.concat(data); - response.body = buffer; - done.resolve(); - }); - await done.promise; - return response as IExtendedIncomingMessage; -}; diff --git a/ts/legacy/smartrequest.formdata.ts b/ts/legacy/smartrequest.formdata.ts deleted file mode 100644 index 5e36c0f..0000000 --- a/ts/legacy/smartrequest.formdata.ts +++ /dev/null @@ -1,99 +0,0 @@ -import * as plugins from './smartrequest.plugins.js'; -import * as interfaces from './smartrequest.interfaces.js'; -import { request } from './smartrequest.request.js'; - -/** - * the interfae for FormFieldData - */ -export interface IFormField { - name: string; - type: 'string' | 'filePath' | 'Buffer'; - payload: string | Buffer; - fileName?: string; - contentType?: string; -} - -const appendFormField = async (formDataArg: plugins.formData, formDataField: IFormField) => { - switch (formDataField.type) { - case 'string': - formDataArg.append(formDataField.name, formDataField.payload); - break; - case 'filePath': - if (typeof formDataField.payload !== 'string') { - throw new Error( - `Payload for key ${ - formDataField.name - } must be of type string. Got ${typeof formDataField.payload} instead.` - ); - } - const fileData = plugins.fs.readFileSync( - plugins.path.join(process.cwd(), formDataField.payload) - ); - formDataArg.append('file', fileData, { - filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf', - contentType: 'application/pdf', - }); - break; - case 'Buffer': - formDataArg.append(formDataField.name, formDataField.payload, { - filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf', - contentType: formDataField.contentType ? formDataField.contentType : 'application/pdf', - }); - break; - } -}; - -export const postFormData = async ( - urlArg: string, - optionsArg: interfaces.ISmartRequestOptions = {}, - payloadArg: IFormField[] -) => { - const form = new plugins.formData(); - for (const formField of payloadArg) { - await appendFormField(form, formField); - } - const requestOptions = { - ...optionsArg, - method: 'POST', - headers: { - ...optionsArg.headers, - ...form.getHeaders(), - }, - requestBody: form, - }; - - // lets fire the actual request for sending the formdata - const response = await request(urlArg, requestOptions); - return response; -}; - -export const postFormDataUrlEncoded = async ( - urlArg: string, - optionsArg: interfaces.ISmartRequestOptions = {}, - payloadArg: { key: string; content: string }[] -) => { - let resultString = ''; - - for (const keyContentPair of payloadArg) { - if (resultString) { - resultString += '&'; - } - resultString += `${encodeURIComponent(keyContentPair.key)}=${encodeURIComponent( - keyContentPair.content - )}`; - } - - const requestOptions: interfaces.ISmartRequestOptions = { - ...optionsArg, - method: 'POST', - headers: { - ...optionsArg.headers, - 'content-type': 'application/x-www-form-urlencoded', - }, - requestBody: resultString, - }; - - // lets fire the actual request for sending the formdata - const response = await request(urlArg, requestOptions); - return response; -}; diff --git a/ts/legacy/smartrequest.interfaces.ts b/ts/legacy/smartrequest.interfaces.ts deleted file mode 100644 index f0acf93..0000000 --- a/ts/legacy/smartrequest.interfaces.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as plugins from './smartrequest.plugins.js'; -import * as https from 'https'; - -export interface ISmartRequestOptions extends https.RequestOptions { - keepAlive?: boolean; - requestBody?: any; - autoJsonParse?: boolean; - queryParams?: { [key: string]: string }; - hardDataCuttingTimeout?: number; -} diff --git a/ts/legacy/smartrequest.jsonrest.ts b/ts/legacy/smartrequest.jsonrest.ts deleted file mode 100644 index d9c2a67..0000000 --- a/ts/legacy/smartrequest.jsonrest.ts +++ /dev/null @@ -1,63 +0,0 @@ -// This file implements methods to get and post JSON in a simple manner. - -import * as interfaces from './smartrequest.interfaces.js'; -import { request } from './smartrequest.request.js'; - -/** - * gets Json and puts the right headers + handles response aggregation - * @param domainArg - * @param optionsArg - */ -export const getJson = async ( - domainArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -) => { - optionsArg.method = 'GET'; - optionsArg.headers = { - ...optionsArg.headers, - }; - let response = await request(domainArg, optionsArg); - return response; -}; - -export const postJson = async ( - domainArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -) => { - optionsArg.method = 'POST'; - if ( - typeof optionsArg.requestBody === 'object' && - (!optionsArg.headers || !optionsArg.headers['Content-Type']) - ) { - // make sure headers exist - if (!optionsArg.headers) { - optionsArg.headers = {}; - } - - // assign the right Content-Type, leaving all other headers in place - optionsArg.headers = { - ...optionsArg.headers, - 'Content-Type': 'application/json', - }; - } - let response = await request(domainArg, optionsArg); - return response; -}; - -export const putJson = async ( - domainArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -) => { - optionsArg.method = 'PUT'; - let response = await request(domainArg, optionsArg); - return response; -}; - -export const delJson = async ( - domainArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -) => { - optionsArg.method = 'DELETE'; - let response = await request(domainArg, optionsArg); - return response; -}; diff --git a/ts/legacy/smartrequest.request.ts b/ts/legacy/smartrequest.request.ts deleted file mode 100644 index ce64345..0000000 --- a/ts/legacy/smartrequest.request.ts +++ /dev/null @@ -1,231 +0,0 @@ -import * as plugins from './smartrequest.plugins.js'; -import * as interfaces from './smartrequest.interfaces.js'; - -export interface IExtendedIncomingMessage extends plugins.http.IncomingMessage { - body: T; -} - -const buildUtf8Response = ( - incomingMessageArg: plugins.http.IncomingMessage, - autoJsonParse = true -): Promise => { - const done = plugins.smartpromise.defer(); - // Continuously update stream with data - let body = ''; - incomingMessageArg.on('data', (chunkArg) => { - body += chunkArg; - }); - - incomingMessageArg.on('end', () => { - if (autoJsonParse) { - try { - (incomingMessageArg as IExtendedIncomingMessage).body = JSON.parse(body); - } catch (err) { - (incomingMessageArg as IExtendedIncomingMessage).body = body; - } - } else { - (incomingMessageArg as IExtendedIncomingMessage).body = body; - } - done.resolve(incomingMessageArg as IExtendedIncomingMessage); - }); - return done.promise; -}; - -/** - * determine wether a url is a unix sock - * @param urlArg - */ -const testForUnixSock = (urlArg: string): boolean => { - const unixRegex = /^(http:\/\/|https:\/\/|)unix:/; - return unixRegex.test(urlArg); -}; - -/** - * determine socketPath and path for unixsock - */ -const parseSocketPathAndRoute = (stringToParseArg: string) => { - const parseRegex = /(.*):(.*)/; - const result = parseRegex.exec(stringToParseArg); - return { - socketPath: result[1], - path: result[2], - }; -}; - -/** - * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls - */ -const httpAgent = new plugins.agentkeepalive({ - keepAlive: true, - maxFreeSockets: 10, - maxSockets: 100, - maxTotalSockets: 1000, - timeout: 60000, -}); - -/** - * a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls - */ -const httpAgentKeepAliveFalse = new plugins.agentkeepalive({ - keepAlive: false, - timeout: 60000, -}); - -/** - * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls - */ -const httpsAgent = new plugins.agentkeepalive.HttpsAgent({ - keepAlive: true, - maxFreeSockets: 10, - maxSockets: 100, - maxTotalSockets: 1000, - timeout: 60000, -}); - -/** - * a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls - */ -const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({ - keepAlive: false, - timeout: 60000, -}); - -export let request = async ( - urlArg: string, - optionsArg: interfaces.ISmartRequestOptions = {}, - responseStreamArg: boolean = false, - requestDataFunc: (req: plugins.http.ClientRequest) => void = null -): Promise => { - const done = plugins.smartpromise.defer(); - - // merge options - const defaultOptions: interfaces.ISmartRequestOptions = { - // agent: agent, - autoJsonParse: true, - keepAlive: true, - }; - - optionsArg = { - ...defaultOptions, - ...optionsArg, - }; - - // parse url - const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(urlArg, { - searchParams: optionsArg.queryParams || {}, - }); - optionsArg.hostname = parsedUrl.hostname; - if (parsedUrl.port) { - optionsArg.port = parseInt(parsedUrl.port, 10); - } - optionsArg.path = parsedUrl.path; - optionsArg.queryParams = parsedUrl.searchParams; - - // determine if unixsock - if (testForUnixSock(urlArg)) { - const detailedUnixPath = parseSocketPathAndRoute(optionsArg.path); - optionsArg.socketPath = detailedUnixPath.socketPath; - optionsArg.path = detailedUnixPath.path; - } - - // TODO: support tcp sockets - - // lets determine agent - switch (true) { - case !!optionsArg.agent: - break; - case parsedUrl.protocol === 'https:' && optionsArg.keepAlive: - optionsArg.agent = httpsAgent; - break; - case parsedUrl.protocol === 'https:' && !optionsArg.keepAlive: - optionsArg.agent = httpsAgentKeepAliveFalse; - break; - case parsedUrl.protocol === 'http:' && optionsArg.keepAlive: - optionsArg.agent = httpAgent; - break; - case parsedUrl.protocol === 'http:' && !optionsArg.keepAlive: - optionsArg.agent = httpAgentKeepAliveFalse; - break; - } - - // lets determine the request module to use - const requestModule = (() => { - switch (true) { - case parsedUrl.protocol === 'https:': - return plugins.https; - case parsedUrl.protocol === 'http:': - return plugins.http; - } - })() as typeof plugins.https; - - if (!requestModule) { - console.error(`The request to ${urlArg} is missing a viable protocol. Must be http or https`); - return; - } - - // lets perform the actual request - const requestToFire = requestModule.request(optionsArg, async (resArg) => { - if (optionsArg.hardDataCuttingTimeout) { - setTimeout(() => { - resArg.destroy(); - done.reject(new Error('Request timed out')); - }, optionsArg.hardDataCuttingTimeout) - } - - if (responseStreamArg) { - done.resolve(resArg as IExtendedIncomingMessage); - } else { - const builtResponse = await buildUtf8Response(resArg, optionsArg.autoJsonParse); - done.resolve(builtResponse); - } - }); - - // lets write the requestBody - if (optionsArg.requestBody) { - if (optionsArg.requestBody instanceof plugins.formData) { - optionsArg.requestBody.pipe(requestToFire).on('finish', (event: any) => { - requestToFire.end(); - }); - } else { - if (typeof optionsArg.requestBody !== 'string') { - optionsArg.requestBody = JSON.stringify(optionsArg.requestBody); - } - requestToFire.write(optionsArg.requestBody); - requestToFire.end(); - } - } else if (requestDataFunc) { - requestDataFunc(requestToFire); - } else { - requestToFire.end(); - } - - // lets handle an error - requestToFire.on('error', (e) => { - console.error(e); - requestToFire.destroy(); - }); - - const response = await done.promise; - response.on('error', (err) => { - console.log(err); - response.destroy(); - }); - return response; -}; - -export const safeGet = async (urlArg: string) => { - const agentToUse = urlArg.startsWith('http://') ? new plugins.http.Agent() : new plugins.https.Agent(); - try { - const response = await request(urlArg, { - method: 'GET', - agent: agentToUse, - timeout: 5000, - hardDataCuttingTimeout: 5000, - autoJsonParse: false, - }); - return response; - } catch (err) { - console.log(err); - return null; - } -}; diff --git a/ts/legacy/smartrequest.stream.ts b/ts/legacy/smartrequest.stream.ts deleted file mode 100644 index 951f298..0000000 --- a/ts/legacy/smartrequest.stream.ts +++ /dev/null @@ -1,17 +0,0 @@ -import * as plugins from './smartrequest.plugins.js'; -import * as interfaces from './smartrequest.interfaces.js'; -import { request } from './smartrequest.request.js'; - -export const getStream = async ( - urlArg: string, - optionsArg: interfaces.ISmartRequestOptions = {} -): Promise => { - try { - // Call the existing request function with responseStreamArg set to true. - const responseStream = await request(urlArg, optionsArg, true); - return responseStream; - } catch (err) { - console.error('An error occurred while getting the stream:', err); - throw err; // Rethrow the error to be handled by the caller. - } -}; \ No newline at end of file diff --git a/ts/modern/features/pagination.ts b/ts/modern/features/pagination.ts index a8424f7..15ead9d 100644 --- a/ts/modern/features/pagination.ts +++ b/ts/modern/features/pagination.ts @@ -1,19 +1,22 @@ -import { type IExtendedIncomingMessage } from '../../legacy/smartrequest.request.js'; +import { type SmartResponse } from '../../core/index.js'; import { type TPaginationConfig, PaginationStrategy, type TPaginatedResponse } from '../types/pagination.js'; /** * Creates a paginated response from a regular response */ -export function createPaginatedResponse( - response: IExtendedIncomingMessage, +export async function createPaginatedResponse( + response: SmartResponse, paginationConfig: TPaginationConfig, queryParams: Record, fetchNextPage: (params: Record) => Promise> -): TPaginatedResponse { +): Promise> { + // Parse response body first + const body = await response.json(); + // Default to response.body for items if response is JSON - let items: T[] = Array.isArray(response.body) - ? response.body - : (response.body?.items || response.body?.data || response.body?.results || []); + let items: T[] = Array.isArray(body) + ? body + : (body?.items || body?.data || body?.results || []); let hasNextPage = false; let nextPageParams: Record = {}; @@ -24,7 +27,7 @@ export function createPaginatedResponse( const config = paginationConfig; const currentPage = parseInt(queryParams[config.pageParam || 'page'] || String(config.startPage || 1)); const limit = parseInt(queryParams[config.limitParam || 'limit'] || String(config.pageSize || 20)); - const total = getValueByPath(response.body, config.totalPath || 'total') || 0; + const total = getValueByPath(body, config.totalPath || 'total') || 0; hasNextPage = currentPage * limit < total; @@ -39,8 +42,8 @@ export function createPaginatedResponse( case PaginationStrategy.CURSOR: { const config = paginationConfig; - const nextCursor = getValueByPath(response.body, config.cursorPath || 'nextCursor'); - const hasMore = getValueByPath(response.body, config.hasMorePath || 'hasMore'); + const nextCursor = getValueByPath(body, config.cursorPath || 'nextCursor'); + const hasMore = getValueByPath(body, config.hasMorePath || 'hasMore'); hasNextPage = !!nextCursor || !!hasMore; diff --git a/ts/modern/index.ts b/ts/modern/index.ts index 8961375..af6a0f4 100644 --- a/ts/modern/index.ts +++ b/ts/modern/index.ts @@ -1,6 +1,9 @@ // Export the main client export { SmartRequestClient } from './smartrequestclient.js'; +// Export response type from core +export { SmartResponse } from '../core/index.js'; + // Export types export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig } from './types/common.js'; export { @@ -34,12 +37,12 @@ export function createFormClient() { * Create a client pre-configured for binary data */ export function createBinaryClient() { - return SmartRequestClient.create().responseType('binary'); + return SmartRequestClient.create().accept('binary'); } /** * Create a client pre-configured for streaming */ export function createStreamClient() { - return SmartRequestClient.create().responseType('stream'); + return SmartRequestClient.create().accept('stream'); } \ No newline at end of file diff --git a/ts/modern/smartrequestclient.ts b/ts/modern/smartrequestclient.ts index bd87702..244c863 100644 --- a/ts/modern/smartrequestclient.ts +++ b/ts/modern/smartrequestclient.ts @@ -1,6 +1,5 @@ -import { type ISmartRequestOptions } from '../legacy/smartrequest.interfaces.js'; -import { request, type IExtendedIncomingMessage } from '../legacy/smartrequest.request.js'; -import * as plugins from '../legacy/smartrequest.plugins.js'; +import { request, SmartResponse, type ICoreRequestOptions } from '../core/index.js'; +import * as plugins from '../core/plugins.js'; import type { HttpMethod, ResponseType, FormField } from './types/common.js'; import { @@ -18,9 +17,7 @@ import { createPaginatedResponse } from './features/pagination.js'; */ export class SmartRequestClient { private _url: string; - private _options: ISmartRequestOptions = {}; - private _responseType: ResponseType = 'json'; - private _timeoutMs: number = 60000; + private _options: ICoreRequestOptions = {}; private _retries: number = 0; private _queryParams: Record = {}; private _paginationConfig?: TPaginationConfig; @@ -94,7 +91,6 @@ export class SmartRequestClient { * Set request timeout in milliseconds */ timeout(ms: number): this { - this._timeoutMs = ms; this._options.timeout = ms; this._options.hardDataCuttingTimeout = ms; return this; @@ -145,16 +141,18 @@ export class SmartRequestClient { } /** - * Set response type + * Set the Accept header to indicate what content type is expected */ - responseType(type: ResponseType): this { - this._responseType = type; - - if (type === 'binary' || type === 'stream') { - this._options.autoJsonParse = false; - } - - return this; + accept(type: ResponseType): this { + // Map response types to Accept header values + const acceptHeaders: Record = { + 'json': 'application/json', + 'text': 'text/plain', + 'binary': 'application/octet-stream', + 'stream': '*/*' + }; + + return this.header('Accept', acceptHeaders[type]); } /** @@ -225,35 +223,35 @@ export class SmartRequestClient { /** * Make a GET request */ - async get(): Promise> { + async get(): Promise> { return this.execute('GET'); } /** * Make a POST request */ - async post(): Promise> { + async post(): Promise> { return this.execute('POST'); } /** * Make a PUT request */ - async put(): Promise> { + async put(): Promise> { return this.execute('PUT'); } /** * Make a DELETE request */ - async delete(): Promise> { + async delete(): Promise> { return this.execute('DELETE'); } /** * Make a PATCH request */ - async patch(): Promise> { + async patch(): Promise> { return this.execute('PATCH'); } @@ -272,7 +270,7 @@ export class SmartRequestClient { const response = await this.execute(); - return createPaginatedResponse( + return await createPaginatedResponse( response, this._paginationConfig, this._queryParams, @@ -298,7 +296,7 @@ export class SmartRequestClient { /** * Execute the HTTP request */ - private async execute(method?: HttpMethod): Promise> { + private async execute(method?: HttpMethod): Promise> { if (method) { this._options.method = method; } @@ -310,28 +308,8 @@ export class SmartRequestClient { for (let attempt = 0; attempt <= this._retries; attempt++) { try { - if (this._responseType === 'stream') { - return await request(this._url, this._options, true) as IExtendedIncomingMessage; - } else if (this._responseType === 'binary') { - const response = await request(this._url, this._options, true); - - // Handle binary response - const dataPromise = plugins.smartpromise.defer(); - const chunks: Buffer[] = []; - - response.on('data', (chunk: Buffer) => chunks.push(chunk)); - response.on('end', () => { - const buffer = Buffer.concat(chunks); - (response as IExtendedIncomingMessage).body = buffer as any; - dataPromise.resolve(); - }); - - await dataPromise.promise; - return response as IExtendedIncomingMessage; - } else { - // Handle JSON or text response - return await request(this._url, this._options) as IExtendedIncomingMessage; - } + const response = await request(this._url, this._options); + return response as SmartResponse; } catch (error) { lastError = error as Error; diff --git a/ts/modern/types/pagination.ts b/ts/modern/types/pagination.ts index 6058580..f5f9422 100644 --- a/ts/modern/types/pagination.ts +++ b/ts/modern/types/pagination.ts @@ -1,4 +1,4 @@ -import { type IExtendedIncomingMessage } from '../../legacy/smartrequest.request.js'; +import { type SmartResponse } from '../../core/index.js'; /** * Pagination strategy options @@ -45,8 +45,8 @@ export interface LinkPaginationConfig { */ export interface CustomPaginationConfig { strategy: PaginationStrategy.CUSTOM; - hasNextPage: (response: IExtendedIncomingMessage) => boolean; - getNextPageParams: (response: IExtendedIncomingMessage, currentParams: Record) => Record; + hasNextPage: (response: SmartResponse) => boolean; + getNextPageParams: (response: SmartResponse, currentParams: Record) => Record; } /** @@ -62,5 +62,5 @@ export interface TPaginatedResponse { hasNextPage: boolean; // Whether there are more pages getNextPage: () => Promise>; // Function to get the next page getAllPages: () => Promise; // Function to get all remaining pages and combine - response: IExtendedIncomingMessage; // Original response + response: SmartResponse; // Original response } \ No newline at end of file