From 9c5a93949982ea7f74b50676615ef2e2f40a42de Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 18 Aug 2025 22:29:24 +0000 Subject: [PATCH] feat(client/smartrequest): Add streaming and raw buffer support to SmartRequest (buffer, stream, raw); update docs and tests --- changelog.md | 11 +++++ readme.md | 94 ++++++++++++++++++++++++++++++++++++++- test/test.streaming.ts | 74 ++++++++++++++++++++++++++++++ ts/00_commitinfo_data.ts | 2 +- ts/client/smartrequest.ts | 68 +++++++++++++++++++++++++++- ts/client/types/common.ts | 6 +++ 6 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 test/test.streaming.ts diff --git a/changelog.md b/changelog.md index 9f3a7af..15adcab 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,16 @@ # Changelog +## 2025-08-18 - 4.3.0 - feat(client/smartrequest) +Add streaming and raw buffer support to SmartRequest (buffer, stream, raw); update docs and tests + +- Add SmartRequest.buffer(data, contentType?) to send Buffer or Uint8Array bodies with Content-Type header. +- Add SmartRequest.stream(stream, contentType?) to accept Node.js Readable streams or web ReadableStream and set Content-Type when provided. +- Add SmartRequest.raw(streamFunc) to allow custom raw streaming functions (Node.js only) and a RawStreamFunction type. +- Wire Node.js stream handling into CoreRequest by passing a requestDataFunc when creating CoreRequest instances. +- Add comprehensive streaming examples and documentation to README describing buffer/stream/raw usage and streaming methods. +- Add tests for streaming behavior (test/test.streaming.ts) covering buffer, stream, raw, and Uint8Array usage. +- Update client exports and plugins to support streaming features and FormData usage where needed. + ## 2025-08-18 - 4.2.2 - fix(client) Fix CI configuration, prevent socket hangs with auto-drain, and apply various client/core TypeScript fixes and test updates diff --git a/readme.md b/readme.md index f625427..6815172 100644 --- a/readme.md +++ b/readme.md @@ -25,7 +25,7 @@ yarn add @push.rocks/smartrequest - ⚡ **Keep-Alive Connections** - Efficient connection pooling in Node.js - 🛡️ **TypeScript First** - Full type safety and IntelliSense support - 🎯 **Zero Magic Defaults** - Explicit configuration following fetch API principles -- 📡 **Streaming Support** - Handle large files and real-time data +- 📡 **Streaming Support** - Stream buffers, files, and custom data without loading into memory - 🔧 **Highly Configurable** - Timeouts, retries, headers, and more ## Architecture @@ -303,6 +303,98 @@ async function uploadMultipleFiles( } ``` +### Streaming Request Bodies + +SmartRequest provides multiple ways to stream data in requests, making it easy to upload large files or send real-time data without loading everything into memory: + +```typescript +import { SmartRequest } from '@push.rocks/smartrequest'; +import * as fs from 'fs'; +import { Readable } from 'stream'; + +// Stream a Buffer directly +async function uploadBuffer() { + const buffer = Buffer.from('Hello, World!'); + + const response = await SmartRequest.create() + .url('https://api.example.com/upload') + .buffer(buffer, 'text/plain') + .post(); + + return await response.json(); +} + +// Stream a file using Node.js streams +async function uploadLargeFile(filePath: string) { + const fileStream = fs.createReadStream(filePath); + + const response = await SmartRequest.create() + .url('https://api.example.com/upload') + .stream(fileStream, 'application/octet-stream') + .post(); + + return await response.json(); +} + +// Stream data from any readable source +async function streamData(dataSource: Readable) { + const response = await SmartRequest.create() + .url('https://api.example.com/stream') + .stream(dataSource) + .post(); + + return await response.json(); +} + +// Advanced: Full control over request streaming (Node.js only) +async function customStreaming() { + const response = await SmartRequest.create() + .url('https://api.example.com/stream') + .raw((request) => { + // Custom streaming logic - you have full control + request.write('chunk1'); + request.write('chunk2'); + + // Stream from another source + someReadableStream.pipe(request); + }) + .post(); + + return await response.json(); +} + +// Send Uint8Array (works in both Node.js and browser) +async function uploadBinaryData() { + const data = new Uint8Array([72, 101, 108, 108, 111]); // "Hello" + + const response = await SmartRequest.create() + .url('https://api.example.com/binary') + .buffer(data, 'application/octet-stream') + .post(); + + return await response.json(); +} +``` + +#### Streaming Methods + +- **`.buffer(data, contentType?)`** - Stream a Buffer or Uint8Array directly + - `data`: Buffer or Uint8Array to send + - `contentType`: Optional content type (defaults to 'application/octet-stream') + +- **`.stream(stream, contentType?)`** - Stream from Node.js ReadableStream or web ReadableStream + - `stream`: The stream to pipe to the request + - `contentType`: Optional content type + +- **`.raw(streamFunc)`** - Advanced control over request streaming (Node.js only) + - `streamFunc`: Function that receives the raw request object for custom streaming + +These methods are particularly useful for: +- Uploading large files without loading them into memory +- Streaming real-time data to servers +- Proxying data between services +- Implementing chunked transfer encoding + ### Unix Socket Support (Node.js only) ```typescript diff --git a/test/test.streaming.ts b/test/test.streaming.ts new file mode 100644 index 0000000..30a118c --- /dev/null +++ b/test/test.streaming.ts @@ -0,0 +1,74 @@ +import { tap, expect } from '@git.zone/tstest/tapbundle'; +import * as fs from 'fs'; +import { SmartRequest } from '../ts/index.js'; + +tap.test('should send a buffer using buffer() method', async () => { + const testBuffer = Buffer.from('Hello, World!'); + + const smartRequest = SmartRequest.create() + .url('https://httpbin.org/post') + .buffer(testBuffer, 'text/plain') + .method('POST'); + + const response = await smartRequest.post(); + const data = await response.json(); + + expect(data).toHaveProperty('data'); + expect(data.data).toEqual('Hello, World!'); + expect(data.headers['Content-Type']).toEqual('text/plain'); +}); + +tap.test('should send a stream using stream() method', async () => { + // Create a simple readable stream + const { Readable } = await import('stream'); + const testData = 'Stream data test'; + const stream = Readable.from([testData]); + + const smartRequest = SmartRequest.create() + .url('https://httpbin.org/post') + .stream(stream, 'text/plain') + .method('POST'); + + const response = await smartRequest.post(); + const data = await response.json(); + + expect(data).toHaveProperty('data'); + expect(data.data).toEqual(testData); +}); + +tap.test('should handle raw streaming with custom function', async () => { + const testData = 'Custom raw stream data'; + + const smartRequest = SmartRequest.create() + .url('https://httpbin.org/post') + .raw((request) => { + // Custom streaming logic + request.write(testData); + request.end(); + }) + .method('POST'); + + const response = await smartRequest.post(); + const data = await response.json(); + + expect(data).toHaveProperty('data'); + expect(data.data).toEqual(testData); +}); + +tap.test('should send Uint8Array using buffer() method', async () => { + const testData = new Uint8Array([72, 101, 108, 108, 111]); // "Hello" in ASCII + + const smartRequest = SmartRequest.create() + .url('https://httpbin.org/post') + .buffer(testData, 'application/octet-stream') + .method('POST'); + + const response = await smartRequest.post(); + const data = await response.json(); + + // Just verify that data was sent + expect(data).toHaveProperty('data'); + expect(data.headers['Content-Type']).toEqual('application/octet-stream'); +}); + +export default tap.start(); \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 02d4874..d89a553 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartrequest', - version: '4.2.2', + version: '4.3.0', description: 'A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.' } diff --git a/ts/client/smartrequest.ts b/ts/client/smartrequest.ts index c928139..64ce276 100644 --- a/ts/client/smartrequest.ts +++ b/ts/client/smartrequest.ts @@ -8,6 +8,7 @@ import type { ResponseType, FormField, RateLimitConfig, + RawStreamFunction, } from './types/common.js'; import { type TPaginationConfig, @@ -121,6 +122,56 @@ export class SmartRequest { return this; } + /** + * Set raw buffer data for the request + */ + buffer(data: Buffer | Uint8Array, contentType?: string): this { + if (!this._options.headers) { + this._options.headers = {}; + } + this._options.headers['Content-Type'] = contentType || 'application/octet-stream'; + this._options.requestBody = data; + return this; + } + + /** + * Stream data for the request + * Accepts Node.js Readable streams or web ReadableStream + */ + stream(stream: NodeJS.ReadableStream | ReadableStream, contentType?: string): this { + if (!this._options.headers) { + this._options.headers = {}; + } + + // Set content type if provided + if (contentType) { + this._options.headers['Content-Type'] = contentType; + } + + // Check if it's a Node.js stream (has pipe method) + if ('pipe' in stream && typeof (stream as any).pipe === 'function') { + // For Node.js streams, we need to use a custom approach + // Store the stream to be used later + (this._options as any).__nodeStream = stream; + } else { + // For web ReadableStream, pass directly + this._options.requestBody = stream; + } + + return this; + } + + /** + * Provide a custom function to handle raw request streaming + * This gives full control over the request body streaming + * Note: Only works in Node.js environment + */ + raw(streamFunc: RawStreamFunction): this { + // Store the raw streaming function to be used later + (this._options as any).__rawStreamFunc = streamFunc; + return this; + } + /** * Set request timeout in milliseconds */ @@ -389,7 +440,22 @@ export class SmartRequest { // Main retry loop for (let attempt = 0; attempt <= this._retries; attempt++) { try { - const request = new CoreRequest(this._url, this._options as any); + // Check if we have a Node.js stream or raw function that needs special handling + let requestDataFunc = null; + if ((this._options as any).__nodeStream) { + const nodeStream = (this._options as any).__nodeStream; + requestDataFunc = (req: any) => { + nodeStream.pipe(req); + }; + // Remove the temporary stream reference + delete (this._options as any).__nodeStream; + } else if ((this._options as any).__rawStreamFunc) { + requestDataFunc = (this._options as any).__rawStreamFunc; + // Remove the temporary function reference + delete (this._options as any).__rawStreamFunc; + } + + const request = new CoreRequest(this._url, this._options as any, requestDataFunc); const response = (await request.fire()) as ICoreResponse; // Check for 429 status if rate limit handling is enabled diff --git a/ts/client/types/common.ts b/ts/client/types/common.ts index 5a3c00a..ac333d6 100644 --- a/ts/client/types/common.ts +++ b/ts/client/types/common.ts @@ -66,3 +66,9 @@ export interface RateLimitConfig { backoffFactor?: number; // Exponential backoff factor (default: 2) onRateLimit?: (attempt: number, waitTime: number) => void; // Callback for rate limit events } + +/** + * Raw streaming function for advanced request body control + * Note: The request parameter type depends on the environment (Node.js ClientRequest or fetch Request) + */ +export type RawStreamFunction = (request: any) => void;