Compare commits

...

43 Commits

Author SHA1 Message Date
bbb57004d9 BREAKING CHANGE(core): major architectural refactoring with fetch-like API
Some checks failed
Default (tags) / security (push) Failing after 24s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-07-27 21:23:20 +00:00
f7d2c6de4f 2.1.0
Some checks failed
Default (tags) / security (push) Failing after 23s
Default (tags) / test (push) Failing after 9s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-04-03 06:52:58 +00:00
b8f545cdd5 feat(docs): Enhance documentation and tests with modern API usage examples and migration guide 2025-04-03 06:52:58 +00:00
96820090d4 add modern version of request construction 2025-04-03 06:36:48 +00:00
6e2c63fe1b 2.0.23 2024-11-06 20:58:17 +01:00
39d3bb4d24 fix(core): Enhance type safety for response in binary requests 2024-11-06 20:58:17 +01:00
62db3a9bc5 update description 2024-05-29 14:15:43 +02:00
a82d9eafe2 2.0.22 2024-04-14 03:54:40 +02:00
f55ab55365 fix(core): update 2024-04-14 03:54:39 +02:00
9cf8c0b0f3 update tsconfig 2024-04-01 21:40:24 +02:00
49796f89dd update npmextra.json: githost 2024-04-01 19:59:25 +02:00
aaeaa23b79 update npmextra.json: githost 2024-03-30 21:48:25 +01:00
85199b8839 2.0.21 2023-11-09 21:22:58 +01:00
33f7df28e2 fix(core): update 2023-11-09 21:22:57 +01:00
29ee46b3a2 2.0.20 2023-11-03 01:37:11 +01:00
18dd110d4e fix(core): update 2023-11-03 01:37:10 +01:00
fa93f13306 2.0.19 2023-11-03 00:47:43 +01:00
81694cf58c fix(core): update 2023-11-03 00:47:42 +01:00
fdd1c7cdb3 2.0.18 2023-07-12 00:53:10 +02:00
40f330791f fix(core): update 2023-07-12 00:53:09 +02:00
16b4d168db 2.0.17 2023-07-12 00:24:37 +02:00
cc017a9bbf fix(core): update 2023-07-12 00:24:36 +02:00
2adf7e9ee2 2.0.16 2023-07-12 00:23:26 +02:00
2303b6da7e fix(core): update 2023-07-12 00:23:25 +02:00
4487579bfd switch to new org scheme 2023-07-10 10:17:43 +02:00
8d2bbcae2a 2.0.15 2023-04-19 14:38:28 +02:00
deb25a3068 fix(core): update 2023-04-19 14:38:28 +02:00
0a83d8b476 2.0.14 2023-04-19 14:24:43 +02:00
8e7c730d86 fix(core): update 2023-04-19 14:24:43 +02:00
fe50adb1ff 2.0.13 2023-04-19 14:13:35 +02:00
cd75f7acd8 fix(core): update 2023-04-19 14:13:34 +02:00
bb0dd6426a 2.0.12 2023-04-19 04:07:44 +02:00
d471376681 fix(core): update 2023-04-19 04:07:44 +02:00
b882922f2b 2.0.11 2022-08-21 14:03:18 +02:00
9a0d35c325 fix(core): update 2022-08-21 14:03:18 +02:00
7b49bba0d2 2.0.10 2022-08-06 22:29:12 +02:00
6600a23a00 fix(core): update 2022-08-06 22:29:12 +02:00
e2845c9992 2.0.9 2022-08-01 17:15:53 +02:00
5e6f2c6fbf fix(core): update 2022-08-01 17:15:52 +02:00
d3d0649b73 2.0.8 2022-08-01 17:10:22 +02:00
fba43df3c4 fix(core): update 2022-08-01 17:10:22 +02:00
c6fa540543 2.0.7 2022-08-01 17:10:11 +02:00
1891b54389 fix(core): update 2022-08-01 17:10:11 +02:00
33 changed files with 12335 additions and 15599 deletions

View File

@@ -0,0 +1,66 @@
name: Default (not tags)
on:
push:
tags-ignore:
- '**'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Install pnpm and npmci
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
- name: Run npm prepare
run: npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build

View File

@@ -0,0 +1,124 @@
name: Default (tags)
on:
push:
tags:
- '*'
env:
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
jobs:
security:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
if: ${{ always() }}
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci npm build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Release
run: |
npmci node install stable
npmci npm publish
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @shipzone/npmci
npmci npm prepare
- name: Code quality
run: |
npmci command npm install -g typescript
npmci npm install
- name: Trigger
run: npmci trigger
- name: Build docs and upload artifacts
run: |
npmci node install stable
npmci npm install
pnpm install -g @git.zone/tsdoc
npmci command tsdoc
continue-on-error: true

View File

@@ -1,128 +0,0 @@
# gitzone ci_default
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
cache:
paths:
- .npmci_cache/
key: '$CI_BUILD_STAGE'
stages:
- security
- test
- release
- metadata
before_script:
- npm install -g @shipzone/npmci
# ====================
# security stage
# ====================
auditProductionDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --production --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=prod --production
tags:
- docker
allow_failure: true
auditDevDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=dev
tags:
- docker
allow_failure: true
# ====================
# test stage
# ====================
testStable:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
testBuild:
stage: test
script:
- npmci npm prepare
- npmci node install stable
- npmci npm install
- npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/
tags:
- docker
release:
stage: release
script:
- npmci node install stable
- npmci npm publish
only:
- tags
tags:
- lossless
- docker
- notpriv
# ====================
# metadata stage
# ====================
codequality:
stage: metadata
allow_failure: true
only:
- tags
script:
- npmci command npm install -g typescript
- npmci npm prepare
- npmci npm install
tags:
- lossless
- docker
- priv
trigger:
stage: metadata
script:
- npmci trigger
only:
- tags
tags:
- lossless
- docker
- notpriv
pages:
stage: metadata
script:
- npmci node install stable
- npmci npm prepare
- npmci npm install
- npmci command npm run buildDocs
tags:
- lossless
- docker
- notpriv
only:
- tags
artifacts:
expire_in: 1 week
paths:
- public
allow_failure: true

98
changelog.md Normal file
View File

@@ -0,0 +1,98 @@
# Changelog
## 2025-07-27 - 3.0.0 - BREAKING CHANGE(core)
Major architectural refactoring with fetch-like API
**Breaking Changes:**
- Legacy API functions are now imported from `@push.rocks/smartrequest/legacy` instead of the main export
- Modern API response objects now use fetch-like methods (`.json()`, `.text()`, `.arrayBuffer()`, `.stream()`) instead of direct `.body` access
- Renamed `responseType()` method to `accept()` in modern API
- Removed automatic defaults:
- No default keepAlive (must be explicitly set)
- No default timeouts
- No automatic JSON parsing in core
- Complete internal architecture refactoring:
- Core module now always returns raw streams
- Response parsing happens in SmartResponse methods
- Legacy API is now just an adapter over the core module
**Features:**
- New fetch-like response API with single-use body consumption
- Better TypeScript support and type safety
- Cleaner separation of concerns between request and response
- More predictable behavior aligned with fetch API standards
**Documentation:**
- Updated all examples to show correct import paths
- Added comprehensive examples for the new response API
- Enhanced migration guide
## 2025-04-03 - 2.1.0 - feat(docs)
Enhance documentation and tests with modern API usage examples and migration guide
- Updated README to include detailed examples for the modern fluent API, covering GET, POST, headers, query, timeout, retries, and pagination
- Added a migration guide comparing the legacy API and modern API usage
- Improved installation instructions with npm, pnpm, and yarn examples
- Added and updated test files for both legacy and modern API functionalities
- Minor formatting improvements in the code and documentation examples
## 2024-11-06 - 2.0.23 - fix(core)
Enhance type safety for response in binary requests
- Updated the dependency versions in package.json to their latest versions.
- Improved type inference for the response body in getBinary method of smartrequest.binaryrest.ts.
- Introduced generic typing to IExtendedIncomingMessage interface for better type safety.
## 2024-05-29 - 2.0.22 - Documentation
update description
## 2024-04-01 - 2.0.21 - Configuration
Updated configuration files
- Updated `tsconfig`
- Updated `npmextra.json`: githost
## 2023-07-10 - 2.0.15 - Structure
Refactored the organization structure
- Switched to a new organization scheme
## 2022-07-29 - 1.1.57 to 2.0.0 - Major Update
Significant changes and improvements leading to a major version update
- **BREAKING CHANGE**: Switched the core to use ECMAScript modules (ESM)
## 2018-08-14 - 1.1.12 to 1.1.13 - Functional Enhancements
Enhanced request capabilities and removed unnecessary dependencies
- Fixed request module to allow sending strings
- Removed CI dependencies
## 2018-07-19 - 1.1.1 to 1.1.11 - Various Fixes and Improvements
Improvements and fixes across various components
- Added formData capability
- Corrected path resolution to use current working directory (CWD)
- Improved formData handling
- Included correct headers
- Updated request ending method
## 2018-06-19 - 1.0.14 - Structural Fix
Resolved conflicts with file extensions
- Changed `.json.ts` to `.jsonrest.ts` to avoid conflicts
## 2018-06-13 - 1.0.8 to 1.0.10 - Core Updates
Ensured binary handling compliance
- Enhanced core to uphold latest standards
- Correct binary file handling response
- Fix for handling and returning binary responses
## 2017-06-09 - 1.0.4 to 1.0.6 - Infrastructure and Type Improvements
Types and infrastructure updates
- Improved types
- Removed need for content type on post requests
- Updated for new infrastructure

View File

@@ -9,12 +9,29 @@
"gitzone": {
"projectType": "npm",
"module": {
"githost": "gitlab.com",
"gitscope": "pushrocks",
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartrequest",
"description": "dropin replacement for request",
"npmPackagename": "@pushrocks/smartrequest",
"license": "MIT"
"description": "A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.",
"npmPackagename": "@push.rocks/smartrequest",
"license": "MIT",
"keywords": [
"HTTP",
"HTTPS",
"request library",
"form data",
"file uploads",
"JSON",
"binary data",
"streams",
"keepAlive",
"TypeScript",
"modern web requests",
"drop-in replacement"
]
}
},
"tsdoc": {
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
}
}

14948
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,10 +1,12 @@
{
"name": "@pushrocks/smartrequest",
"version": "2.0.6",
"name": "@push.rocks/smartrequest",
"version": "3.0.0",
"private": false,
"description": "dropin replacement for request",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"description": "A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.",
"exports": {
".": "./dist_ts_web/index.js",
"./legacy": "./dist_ts/legacy/index.js"
},
"type": "module",
"scripts": {
"test": "(tstest test/ --web)",
@@ -13,29 +15,40 @@
},
"repository": {
"type": "git",
"url": "git+ssh://git@gitlab.com/pushrocks/smartrequest.git"
"url": "https://code.foss.global/push.rocks/smartrequest.git"
},
"keywords": [
"request"
"HTTP",
"HTTPS",
"request library",
"form data",
"file uploads",
"JSON",
"binary data",
"streams",
"keepAlive",
"TypeScript",
"modern web requests",
"drop-in replacement"
],
"author": "Lossless GmbH",
"author": "Task Venture Capital GmbH",
"license": "MIT",
"bugs": {
"url": "https://gitlab.com/pushrocks/smartrequest/issues"
"url": "https://gitlab.com/push.rocks/smartrequest/issues"
},
"homepage": "https://gitlab.com/pushrocks/smartrequest#README",
"homepage": "https://code.foss.global/push.rocks/smartrequest",
"dependencies": {
"@pushrocks/smartpromise": "^3.1.7",
"@pushrocks/smarturl": "^3.0.2",
"agentkeepalive": "^4.2.1",
"form-data": "^4.0.0"
"@push.rocks/smartpromise": "^4.0.4",
"@push.rocks/smarturl": "^3.1.0",
"agentkeepalive": "^4.5.0",
"form-data": "^4.0.1"
},
"devDependencies": {
"@gitzone/tsbuild": "^2.1.63",
"@gitzone/tsrun": "^1.2.37",
"@gitzone/tstest": "^1.0.72",
"@pushrocks/tapbundle": "^5.0.4",
"@types/node": "^18.6.2"
"@git.zone/tsbuild": "^2.2.0",
"@git.zone/tsrun": "^1.3.3",
"@git.zone/tstest": "^1.0.90",
"@pushrocks/tapbundle": "^5.0.8",
"@types/node": "^22.9.0"
},
"files": [
"ts/**/*",
@@ -51,5 +64,6 @@
],
"browserslist": [
"last 1 chrome versions"
]
],
"packageManager": "pnpm@10.7.0+sha512.6b865ad4b62a1d9842b61d674a393903b871d9244954f652b8842c2b553c72176b278f64c463e52d40fff8aba385c235c8c9ecf5cc7de4fd78b8bb6d49633ab6"
}

9960
pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

54
readme.hints.md Normal file
View File

@@ -0,0 +1,54 @@
# SmartRequest Architecture Hints
## Core Features
- supports http
- supports https
- supports unix socks
- supports formData
- supports file uploads
- supports best practice keepAlive
- dedicated functions for working with JSON request/response cycles
- written in TypeScript
- continuously updated
- uses node native http and https modules
- used in modules like @push.rocks/smartproxy and @api.global/typedrequest
## Architecture Overview (as of latest refactoring)
- The project is now structured with a clean separation between core functionality and API layers
- Core module (ts/core/) contains the essential HTTP request logic using Node.js http/https modules
- **Core always returns raw streams** - no parsing or body collection happens in the core request function
- Modern API (ts/modern/) provides a fluent, chainable interface with fetch-like Response objects
- Legacy API is maintained through a thin adapter layer for backward compatibility
## Key Components
### Core Module (ts/core/)
- `request.ts`: Core HTTP/HTTPS request logic with unix socket support and keep-alive agents
- `coreRequest()` always returns a raw Node.js IncomingMessage stream
- No response parsing or body collection happens here
- `response.ts`: SmartResponse class providing fetch-like API
- Methods like `json()`, `text()`, `arrayBuffer()` handle all parsing and body collection
- Response body is streamed and collected only when these methods are called
- Body can only be consumed once (throws error on second attempt)
- `types.ts`: Core TypeScript interfaces and types
- `plugins.ts`: Centralized dependencies
### Modern API
- SmartRequestClient: Fluent API with method chaining
- Returns SmartResponse objects with fetch-like methods
- Supports pagination, retries, timeouts, and various response types
### Binary Request Handling
- Binary requests are handled correctly when `responseType: 'binary'` is set
- Response body is kept as Buffer without string conversion
- No automatic transformations applied to binary data
### Legacy Compatibility
- All legacy functions (getJson, postJson, etc.) are maintained through adapter.ts
- Legacy API returns IExtendedIncomingMessage for backward compatibility
- Modern API can be accessed alongside legacy API
## Testing
- Use `pnpm test` to run all tests
- Modern API tests use the new SmartResponse methods (response.json(), response.text())
- Legacy API tests continue to use the body property directly

618
readme.md
View File

@@ -1,96 +1,558 @@
# @pushrocks/smartrequest
dropin replacement for request
# @push.rocks/smartrequest
A modern HTTP/HTTPS request library for Node.js with support for form data, file uploads, JSON, binary data, streams, and unix sockets. Features both a legacy API for backward compatibility and a modern fetch-like API for new projects.
## Availabililty and Links
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartrequest)
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartrequest)
* [github.com (source mirror)](https://github.com/pushrocks/smartrequest)
* [docs (typedoc)](https://pushrocks.gitlab.io/smartrequest/)
## Install
To install `@push.rocks/smartrequest`, use one of the following commands:
## Status for master
```bash
# Using npm
npm install @push.rocks/smartrequest --save
Status Category | Status Badge
-- | --
GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartrequest/badges/master/pipeline.svg)](https://lossless.cloud)
GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartrequest/badges/master/coverage.svg)](https://lossless.cloud)
npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartrequest)](https://lossless.cloud)
Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartrequest)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartrequest)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartrequest)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartrequest)](https://lossless.cloud)
Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
# Using pnpm
pnpm add @push.rocks/smartrequest
# Using yarn
yarn add @push.rocks/smartrequest
```
This will add `@push.rocks/smartrequest` to your project's dependencies.
## Key Features
- 🚀 **Modern Fetch-like API** - Familiar response methods (`.json()`, `.text()`, `.arrayBuffer()`, `.stream()`)
- 🔄 **Two API Styles** - Legacy function-based API and modern fluent chainable API
- 🌐 **Unix Socket Support** - Connect to local services like Docker
- 📦 **Form Data & File Uploads** - Built-in support for multipart/form-data
- 🔁 **Pagination Support** - Multiple strategies (offset, cursor, Link headers)
-**Keep-Alive Connections** - Efficient connection pooling
- 🛡️ **TypeScript First** - Full type safety and IntelliSense support
- 🎯 **Zero Magic Defaults** - Explicit configuration following fetch API principles
- 🔌 **Streaming Support** - Handle large files and real-time data
- 🔧 **Highly Configurable** - Timeouts, retries, headers, and more
## Usage
Use TypeScript for best in class instellisense.
`@push.rocks/smartrequest` is designed as a versatile, modern HTTP client library for making HTTP/HTTPS requests in Node.js environments. It provides a clean, type-safe API inspired by the native fetch API but with additional features needed for server-side applications.
### Features
The library provides two distinct APIs:
- supports http
- supports https
- supports unix socks
- supports formData
- supports file uploads
- supports best practice keepAlive
- dedicated functions for working with JSON request/response cycles
- written in TypeScript
- continuously updated
- uses node native http and https modules
- used in modules like @pushrocks/smartproxy and @apiglobal/typedrequest
- commercial support available at [https://lossless.support](https://lossless.support)
1. **Legacy API** - Simple function-based API for quick requests and backward compatibility
2. **Modern Fluent API** - A chainable, fetch-like API for more complex scenarios
> note: smartrequest uses the **native** node http/https modules under the hood (not the bloated one from npm)
Below we will cover key usage scenarios of `@push.rocks/smartrequest`, showcasing its capabilities and providing you with a solid starting point to integrate it into your projects.
```javascript
import * as smartrequest from 'smartrequest'
### Import Guide
// simple post
const options: smartrequest.ISmartRequestOptions = { // typed options
headers: {
"Content-Type": "application/json"
"Authorization": "Bearer token"
},
requestBody: JSON.stringify({
key1: 'value1',
key2: 3
})
}
```typescript
// Modern API (recommended for new projects)
import { SmartRequestClient } from '@push.rocks/smartrequest';
smartrequest.request('https://example.com', options).then(res => {
console.log(res.status)
console.log(res.body) // if json, body will be parsed automatically
}).catch(err => {
console.log(err)
})
// dedicated JSON methods are available:
smartrequest.getJson(...)
smartrequest.postJson(...)
smartrequest.putJson(...)
smartrequest.delJson(...)
// streaming
smartrequest.get('https://example.com/bigfile.mp4', optionsArg, true).then(res => { // third arg = true signals streaming
console.log(res.status)
res.on('data', data => {
// do something with the data chunk here
}
res.on('end', () => {
// do something when things have ended
})
})
// Legacy API (for backward compatibility)
import { getJson, postJson, request } from '@push.rocks/smartrequest/legacy';
```
## Contribution
### Simple GET Request
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
For fetching data from a REST API or any web service that returns JSON:
For further information read the linked docs at the top of this readme.
```typescript
import { getJson } from '@push.rocks/smartrequest/legacy';
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
async function fetchGitHubUserInfo(username: string) {
const response = await getJson(`https://api.github.com/users/${username}`);
console.log(response.body); // The body contains the JSON response
}
[![repo-footer](https://lossless.gitlab.io/publicrelations/repofooter.svg)](https://maintainedby.lossless.com)
fetchGitHubUserInfo('octocat');
```
The `getJson` function simplifies the process of sending a GET request and parsing the JSON response.
### POST Requests with JSON
When you need to send JSON data to a server, for example, creating a new resource:
```typescript
import { postJson } from '@push.rocks/smartrequest/legacy';
async function createTodoItem(todoDetails: { title: string; completed: boolean }) {
const response = await postJson('https://jsonplaceholder.typicode.com/todos', {
requestBody: todoDetails
});
console.log(response.body); // Log the created todo item
}
createTodoItem({ title: 'Implement smartrequest', completed: false });
```
`postJson` handles setting the appropriate content-type header and stringifies the JSON body.
### Handling Form Data and File Uploads
`@push.rocks/smartrequest` simplifies the process of uploading files and submitting form data to a server:
```typescript
import { postFormData, IFormField } from '@push.rocks/smartrequest/legacy';
async function uploadProfilePicture(formDataFields: IFormField[]) {
await postFormData('https://api.example.com/upload', {}, formDataFields);
}
uploadProfilePicture([
{ name: 'avatar', type: 'filePath', payload: './path/to/avatar.jpg', fileName: 'avatar.jpg', contentType: 'image/jpeg' },
{ name: 'user_id', type: 'string', payload: '12345' }
]);
```
### Streaming Support
For cases when dealing with large datasets or streaming APIs, `@push.rocks/smartrequest` provides streaming capabilities:
```typescript
import { getStream } from '@push.rocks/smartrequest/legacy';
async function streamLargeFile(url: string) {
const stream = await getStream(url);
stream.on('data', (chunk) => {
console.log('Received chunk of data.');
});
stream.on('end', () => {
console.log('Stream ended.');
});
}
streamLargeFile('https://example.com/largefile');
```
`getStream` allows you to handle data as it's received, which can be beneficial for performance and scalability.
### Advanced Options and Customization
`@push.rocks/smartrequest` is built to be flexible, allowing you to specify additional options to tailor requests to your needs:
```typescript
import { request, ISmartRequestOptions } from '@push.rocks/smartrequest/legacy';
async function customRequestExample() {
const options: ISmartRequestOptions = {
method: 'GET',
headers: {
'Custom-Header': 'Value'
},
keepAlive: true // Enables connection keep-alive
};
const response = await request('https://example.com/data', options);
console.log(response.body);
}
customRequestExample();
```
`request` is the underlying function that powers the simpler `getJson`, `postJson`, etc., and provides you with full control over the HTTP request.
## Modern Fluent API
In addition to the legacy API shown above, `@push.rocks/smartrequest` provides a modern, fluent API with a fetch-like response interface that offers a more chainable and TypeScript-friendly approach to making HTTP requests.
### Basic Usage with the Modern API
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
// Simple GET request
async function fetchUserData(userId: number) {
const response = await SmartRequestClient.create()
.url(`https://jsonplaceholder.typicode.com/users/${userId}`)
.get();
// Use the fetch-like response API
const userData = await response.json();
console.log(userData); // The parsed JSON response
}
// POST request with JSON body
async function createPost(title: string, body: string, userId: number) {
const response = await SmartRequestClient.create()
.url('https://jsonplaceholder.typicode.com/posts')
.json({ title, body, userId })
.post();
const createdPost = await response.json();
console.log(createdPost); // The created post
}
```
### Setting Headers and Query Parameters
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
async function searchRepositories(query: string, perPage: number = 10) {
const response = await SmartRequestClient.create()
.url('https://api.github.com/search/repositories')
.header('Accept', 'application/vnd.github.v3+json')
.query({
q: query,
per_page: perPage.toString()
})
.get();
const data = await response.json();
return data.items;
}
```
### Handling Timeouts and Retries
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
async function fetchWithRetry(url: string) {
const response = await SmartRequestClient.create()
.url(url)
.timeout(5000) // 5 seconds timeout
.retry(3) // Retry up to 3 times on failure
.get();
return await response.json();
}
```
### Working with Different Response Types
The modern API provides a fetch-like interface for handling different response types:
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
// JSON response (default)
async function fetchJson(url: string) {
const response = await SmartRequestClient.create()
.url(url)
.get();
return await response.json(); // Parses JSON automatically
}
// Text response
async function fetchText(url: string) {
const response = await SmartRequestClient.create()
.url(url)
.get();
return await response.text(); // Returns response as string
}
// Binary data
async function downloadImage(url: string) {
const response = await SmartRequestClient.create()
.url(url)
.accept('binary') // Optional: hints to server we want binary
.get();
const buffer = await response.arrayBuffer();
return Buffer.from(buffer); // Convert ArrayBuffer to Buffer if needed
}
// Streaming response
async function streamLargeFile(url: string) {
const response = await SmartRequestClient.create()
.url(url)
.get();
// Get the underlying Node.js stream
const stream = response.stream();
stream.on('data', (chunk) => {
console.log(`Received ${chunk.length} bytes of data`);
});
return new Promise((resolve, reject) => {
stream.on('end', resolve);
stream.on('error', reject);
});
}
```
### Pagination Support
The modern API includes built-in support for various pagination strategies:
```typescript
import { SmartRequestClient, PaginationStrategy } from '@push.rocks/smartrequest';
// Offset-based pagination (page & limit)
async function fetchAllUsers() {
const client = SmartRequestClient.create()
.url('https://api.example.com/users')
.withOffsetPagination({
pageParam: 'page',
limitParam: 'limit',
startPage: 1,
pageSize: 20,
totalPath: 'meta.total'
});
// Get first page with pagination info
const firstPage = await client.getPaginated();
console.log(`Found ${firstPage.items.length} users on first page`);
console.log(`Has more pages: ${firstPage.hasNextPage}`);
if (firstPage.hasNextPage) {
// Get next page
const secondPage = await firstPage.getNextPage();
console.log(`Found ${secondPage.items.length} more users`);
}
// Or get all pages at once (use with caution for large datasets)
const allUsers = await client.getAllPages();
console.log(`Retrieved ${allUsers.length} users in total`);
}
// Cursor-based pagination
async function fetchAllPosts() {
const allPosts = await SmartRequestClient.create()
.url('https://api.example.com/posts')
.withCursorPagination({
cursorParam: 'cursor',
cursorPath: 'meta.nextCursor',
hasMorePath: 'meta.hasMore'
})
.getAllPages();
console.log(`Retrieved ${allPosts.length} posts in total`);
}
// Link header-based pagination (GitHub API style)
async function fetchAllIssues(repo: string) {
const paginatedResponse = await SmartRequestClient.create()
.url(`https://api.github.com/repos/${repo}/issues`)
.header('Accept', 'application/vnd.github.v3+json')
.withLinkPagination()
.getPaginated();
return paginatedResponse.getAllPages();
}
```
### Advanced Features
#### Unix Socket Support
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
// Connect to a service via Unix socket
async function queryViaUnixSocket() {
const response = await SmartRequestClient.create()
.url('http://unix:/var/run/docker.sock:/v1.24/containers/json')
.get();
return await response.json();
}
```
#### Form Data with File Uploads
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
async function uploadMultipleFiles(files: Array<{name: string, path: string}>) {
const formFields = files.map(file => ({
name: 'files',
value: fs.readFileSync(file.path),
filename: file.name,
contentType: 'application/octet-stream'
}));
const response = await SmartRequestClient.create()
.url('https://api.example.com/upload')
.formData(formFields)
.post();
return await response.json();
}
```
#### Keep-Alive Connections
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
// Enable keep-alive for better performance with multiple requests
async function performMultipleRequests() {
const client = SmartRequestClient.create()
.header('Connection', 'keep-alive');
// Requests will reuse the same connection
const results = await Promise.all([
client.url('https://api.example.com/endpoint1').get(),
client.url('https://api.example.com/endpoint2').get(),
client.url('https://api.example.com/endpoint3').get()
]);
return Promise.all(results.map(r => r.json()));
}
```
### Response Object Methods
The modern API returns a `SmartResponse` object with the following methods:
- `json<T>(): Promise<T>` - Parse response as JSON
- `text(): Promise<string>` - Get response as text
- `arrayBuffer(): Promise<ArrayBuffer>` - Get response as ArrayBuffer
- `stream(): NodeJS.ReadableStream` - Get the underlying Node.js stream
- `raw(): http.IncomingMessage` - Get the raw http.IncomingMessage
Each body method can only be called once per response, similar to the fetch API.
Through its comprehensive set of features tailored for modern web development, `@push.rocks/smartrequest` aims to provide developers with a powerful tool for handling HTTP/HTTPS requests efficiently. Whether it's a simple API call, handling form data, processing streams, or working with paginated APIs, `@push.rocks/smartrequest` delivers a robust, type-safe solution to fit your project's requirements.
## Migration Guide: Legacy API to Modern API
If you're currently using the legacy API and want to migrate to the modern fluent API, here's a quick reference guide:
| Legacy API | Modern API |
|------------|------------|
| `getJson(url)` | `SmartRequestClient.create().url(url).get()` |
| `postJson(url, { requestBody: data })` | `SmartRequestClient.create().url(url).json(data).post()` |
| `putJson(url, { requestBody: data })` | `SmartRequestClient.create().url(url).json(data).put()` |
| `delJson(url)` | `SmartRequestClient.create().url(url).delete()` |
| `postFormData(url, {}, fields)` | `SmartRequestClient.create().url(url).formData(fields).post()` |
| `getStream(url)` | `SmartRequestClient.create().url(url).accept('stream').get()` |
| `request(url, options)` | `SmartRequestClient.create().url(url).[...configure options...].get()` |
The modern API provides more flexibility and better TypeScript integration, making it the recommended approach for new projects.
## Complete Examples
### Building a REST API Client
Here's a complete example of building a typed API client using smartrequest:
```typescript
import { SmartRequestClient, type SmartResponse } from '@push.rocks/smartrequest';
interface User {
id: number;
name: string;
email: string;
}
interface Post {
id: number;
title: string;
body: string;
userId: number;
}
class BlogApiClient {
private baseUrl = 'https://jsonplaceholder.typicode.com';
private async request(path: string) {
return SmartRequestClient.create()
.url(`${this.baseUrl}${path}`)
.header('Accept', 'application/json');
}
async getUser(id: number): Promise<User> {
const response = await this.request(`/users/${id}`).get();
return response.json<User>();
}
async createPost(post: Omit<Post, 'id'>): Promise<Post> {
const response = await this.request('/posts')
.json(post)
.post();
return response.json<Post>();
}
async deletePost(id: number): Promise<void> {
const response = await this.request(`/posts/${id}`).delete();
if (!response.ok) {
throw new Error(`Failed to delete post: ${response.statusText}`);
}
}
async getAllPosts(userId?: number): Promise<Post[]> {
const client = this.request('/posts');
if (userId) {
client.query({ userId: userId.toString() });
}
const response = await client.get();
return response.json<Post[]>();
}
}
// Usage
const api = new BlogApiClient();
const user = await api.getUser(1);
const posts = await api.getAllPosts(user.id);
```
### Error Handling
```typescript
import { SmartRequestClient } from '@push.rocks/smartrequest';
async function fetchWithErrorHandling(url: string) {
try {
const response = await SmartRequestClient.create()
.url(url)
.timeout(5000)
.retry(2)
.get();
// Check if request was successful
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
// Handle different content types
const contentType = response.headers['content-type'];
if (contentType?.includes('application/json')) {
return await response.json();
} else if (contentType?.includes('text/')) {
return await response.text();
} else {
return await response.arrayBuffer();
}
} catch (error) {
if (error.code === 'ECONNREFUSED') {
console.error('Connection refused - is the server running?');
} else if (error.code === 'ETIMEDOUT') {
console.error('Request timed out');
} else {
console.error('Request failed:', error.message);
}
throw error;
}
}
```
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

53
readme.plan.md Normal file
View File

@@ -0,0 +1,53 @@
# Smartrequest Refactoring Plan
Command to reread CLAUDE.md: `cat /home/philkunz/.claude/CLAUDE.md`
## Objective
Refactor smartrequest to use native fetch-like API with a streamlined core that supports unix sockets and keep-alive.
## Architecture Overview
- Rename `legacy/` to `core/` and remove "smartrequest." prefix from filenames
- Create a modern Response class similar to fetch API
- Use core as foundation for modern API, not as legacy adapter
- Maintain unix socket and keep-alive support
## Task Checklist
- [x] Reread /home/philkunz/.claude/CLAUDE.md
- [x] Create ts/core directory structure with request.ts, types.ts, and plugins.ts
- [x] Migrate core request logic from legacy to core/request.ts
- [x] Create modern Response class with fetch-like API
- [x] Update modern API to use new core module
- [x] Create legacy adapter for backward compatibility
- [x] Update exports in ts/index.ts
- [x] Run tests and fix any issues
- [x] Clean up old legacy files
## Implementation Details
### Core Module Structure
```
ts/core/
├── request.ts # Core HTTP/HTTPS request logic with unix socket support
├── types.ts # Core interfaces and types
├── plugins.ts # Dependencies (http, https, agentkeepalive, etc.)
└── response.ts # Modern Response class
```
### Response Class API
The new Response class will provide fetch-like methods:
- `json()`: Promise<T> - Parse response as JSON
- `text()`: Promise<string> - Get response as text
- `arrayBuffer()`: Promise<ArrayBuffer> - Get response as ArrayBuffer
- `stream()`: ReadableStream - Get response as stream
- `ok`: boolean - Status is 2xx
- `status`: number - HTTP status code
- `statusText`: string - HTTP status text
- `headers`: Headers - Response headers
### Migration Strategy
1. Move core request logic without breaking changes
2. Create Response wrapper that provides modern API
3. Update SmartRequestClient to use new core
4. Add legacy adapter for backward compatibility
5. Ensure all tests pass throughout migration

View File

@@ -1,6 +1,6 @@
import { tap, expect, expectAsync } from '@pushrocks/tapbundle';
import * as smartrequest from '../ts/index.js';
import * as smartrequest from '../ts/legacy/index.js';
tap.test('should request a html document over https', async () => {
await expectAsync(smartrequest.getJson('https://encrypted.google.com/')).toHaveProperty('body');
@@ -13,11 +13,18 @@ tap.test('should request a JSON document over https', async () => {
.toEqual(1);
});
tap.skip.test('should post a JSON document over http', async () => {
await expectAsync(smartrequest.postJson('http://md5.jsontest.com/?text=example_text'))
tap.test('should post a JSON document over http', async () => {
const testData = { text: 'example_text' };
await expectAsync(smartrequest.postJson('https://httpbin.org/post', { requestBody: testData }))
.property('body')
.property('md5')
.toEqual('fa4c6baa0812e5b5c80ed8885e55a8a6');
.property('json')
.property('text')
.toEqual('example_text');
});
tap.test('should safe get stuff', async () => {
smartrequest.safeGet('http://coffee.link/');
smartrequest.safeGet('https://coffee.link/');
});
tap.skip.test('should deal with unix socks', async () => {

95
test/test.modern.ts Normal file
View File

@@ -0,0 +1,95 @@
import { tap, expect } from '@pushrocks/tapbundle';
import { SmartRequestClient } from '../ts/modern/index.js';
tap.test('modern: should request a html document over https', async () => {
const response = await SmartRequestClient.create()
.url('https://encrypted.google.com/')
.get();
expect(response).not.toBeNull();
expect(response).toHaveProperty('status');
expect(response.status).toBeGreaterThan(0);
const text = await response.text();
expect(text.length).toBeGreaterThan(0);
});
tap.test('modern: should request a JSON document over https', async () => {
const response = await SmartRequestClient.create()
.url('https://jsonplaceholder.typicode.com/posts/1')
.get();
const body = await response.json();
expect(body).toHaveProperty('id');
expect(body.id).toEqual(1);
});
tap.test('modern: should post a JSON document over http', async () => {
const testData = { text: 'example_text' };
const response = await SmartRequestClient.create()
.url('https://httpbin.org/post')
.json(testData)
.post();
const body = await response.json();
expect(body).toHaveProperty('json');
expect(body.json).toHaveProperty('text');
expect(body.json.text).toEqual('example_text');
});
tap.test('modern: should set headers correctly', async () => {
const customHeader = 'X-Custom-Header';
const headerValue = 'test-value';
const response = await SmartRequestClient.create()
.url('https://httpbin.org/headers')
.header(customHeader, headerValue)
.get();
const body = await response.json();
expect(body).toHaveProperty('headers');
// Check if the header exists (case-sensitive)
expect(body.headers).toHaveProperty(customHeader);
expect(body.headers[customHeader]).toEqual(headerValue);
});
tap.test('modern: should handle query parameters', async () => {
const params = { param1: 'value1', param2: 'value2' };
const response = await SmartRequestClient.create()
.url('https://httpbin.org/get')
.query(params)
.get();
const body = await response.json();
expect(body).toHaveProperty('args');
expect(body.args).toHaveProperty('param1');
expect(body.args.param1).toEqual('value1');
expect(body.args).toHaveProperty('param2');
expect(body.args.param2).toEqual('value2');
});
tap.test('modern: should handle timeout configuration', async () => {
// This test just verifies that the timeout method doesn't throw
const client = SmartRequestClient.create()
.url('https://httpbin.org/get')
.timeout(5000);
const response = await client.get();
expect(response).toHaveProperty('ok');
expect(response.ok).toBeTrue();
});
tap.test('modern: should handle retry configuration', async () => {
// This test just verifies that the retry method doesn't throw
const client = SmartRequestClient.create()
.url('https://httpbin.org/get')
.retry(1);
const response = await client.get();
expect(response).toHaveProperty('ok');
expect(response.ok).toBeTrue();
});
tap.start();

View File

@@ -1,8 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@pushrocks/smartrequest',
version: '2.0.6',
description: 'dropin replacement for request'
name: '@push.rocks/smartrequest',
version: '2.1.0',
description: 'A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.'
}

4
ts/core/index.ts Normal file
View File

@@ -0,0 +1,4 @@
// Core exports
export * from './types.js';
export * from './response.js';
export { request, coreRequest, isUnixSocket, parseUnixSocketUrl } from './request.js';

View File

@@ -7,8 +7,8 @@ import * as path from 'path';
export { http, https, fs, path };
// pushrocks scope
import * as smartpromise from '@pushrocks/smartpromise';
import * as smarturl from '@pushrocks/smarturl';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smarturl from '@push.rocks/smarturl';
export { smartpromise, smarturl };
@@ -16,4 +16,4 @@ export { smartpromise, smarturl };
import agentkeepalive from 'agentkeepalive';
import formData from 'form-data';
export { agentkeepalive, formData };
export { agentkeepalive, formData };

159
ts/core/request.ts Normal file
View File

@@ -0,0 +1,159 @@
import * as plugins from './plugins.js';
import * as types from './types.js';
import { SmartResponse } from './response.js';
// Keep-alive agents for connection pooling
const httpAgent = new plugins.agentkeepalive({
keepAlive: true,
maxFreeSockets: 10,
maxSockets: 100,
maxTotalSockets: 1000,
});
const httpAgentKeepAliveFalse = new plugins.agentkeepalive({
keepAlive: false,
});
const httpsAgent = new plugins.agentkeepalive.HttpsAgent({
keepAlive: true,
maxFreeSockets: 10,
maxSockets: 100,
maxTotalSockets: 1000,
});
const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({
keepAlive: false,
});
/**
* Tests if a URL is a unix socket
*/
export const isUnixSocket = (url: string): boolean => {
const unixRegex = /^(http:\/\/|https:\/\/|)unix:/;
return unixRegex.test(url);
};
/**
* Parses socket path and route from unix socket URL
*/
export const parseUnixSocketUrl = (url: string): { socketPath: string; path: string } => {
const parseRegex = /(.*):(.*)/;
const result = parseRegex.exec(url);
return {
socketPath: result[1],
path: result[2],
};
};
/**
* Core request function that handles all HTTP/HTTPS requests
*/
export async function coreRequest(
urlArg: string,
optionsArg: types.ICoreRequestOptions = {},
requestDataFunc: ((req: plugins.http.ClientRequest) => void) | null = null
): Promise<plugins.http.IncomingMessage> {
const done = plugins.smartpromise.defer<plugins.http.IncomingMessage>();
// No defaults - let users explicitly set options to match fetch behavior
// Parse URL
const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(urlArg, {
searchParams: optionsArg.queryParams || {},
});
optionsArg.hostname = parsedUrl.hostname;
if (parsedUrl.port) {
optionsArg.port = parseInt(parsedUrl.port, 10);
}
optionsArg.path = parsedUrl.path;
// Handle unix socket URLs
if (isUnixSocket(urlArg)) {
const { socketPath, path } = parseUnixSocketUrl(optionsArg.path);
optionsArg.socketPath = socketPath;
optionsArg.path = path;
}
// Determine agent based on protocol and keep-alive setting
if (!optionsArg.agent) {
// Only use keep-alive agents if explicitly requested
if (optionsArg.keepAlive === true) {
optionsArg.agent = parsedUrl.protocol === 'https:' ? httpsAgent : httpAgent;
} else if (optionsArg.keepAlive === false) {
optionsArg.agent = parsedUrl.protocol === 'https:' ? httpsAgentKeepAliveFalse : httpAgentKeepAliveFalse;
}
// If keepAlive is undefined, don't set any agent (more fetch-like behavior)
}
// Determine request module
const requestModule = parsedUrl.protocol === 'https:' ? plugins.https : plugins.http;
if (!requestModule) {
throw new Error(`The request to ${urlArg} is missing a viable protocol. Must be http or https`);
}
// Perform the request
const request = requestModule.request(optionsArg, async (response) => {
// Handle hard timeout
if (optionsArg.hardDataCuttingTimeout) {
setTimeout(() => {
response.destroy();
done.reject(new Error('Request timed out'));
}, optionsArg.hardDataCuttingTimeout);
}
// Always return the raw stream
done.resolve(response);
});
// Write request body
if (optionsArg.requestBody) {
if (optionsArg.requestBody instanceof plugins.formData) {
optionsArg.requestBody.pipe(request).on('finish', () => {
request.end();
});
} else {
// Write body as-is - caller is responsible for serialization
const bodyData = typeof optionsArg.requestBody === 'string'
? optionsArg.requestBody
: optionsArg.requestBody instanceof Buffer
? optionsArg.requestBody
: JSON.stringify(optionsArg.requestBody); // Still stringify for backward compatibility
request.write(bodyData);
request.end();
}
} else if (requestDataFunc) {
requestDataFunc(request);
} else {
request.end();
}
// Handle request errors
request.on('error', (e) => {
console.error(e);
request.destroy();
done.reject(e);
});
// Get response and handle response errors
const response = await done.promise;
response.on('error', (err) => {
console.error(err);
response.destroy();
});
return response;
}
/**
* Modern request function that returns a SmartResponse
*/
export async function request(
urlArg: string,
optionsArg: types.ICoreRequestOptions = {}
): Promise<SmartResponse> {
const response = await coreRequest(urlArg, optionsArg);
return new SmartResponse(response, urlArg);
}

110
ts/core/response.ts Normal file
View File

@@ -0,0 +1,110 @@
import * as plugins from './plugins.js';
import * as types from './types.js';
/**
* Modern Response class that provides a fetch-like API
*/
export class SmartResponse<T = any> implements types.ICoreResponse<T> {
private incomingMessage: plugins.http.IncomingMessage;
private bodyBufferPromise: Promise<Buffer> | null = null;
private consumed = false;
// Public properties
public readonly ok: boolean;
public readonly status: number;
public readonly statusText: string;
public readonly headers: plugins.http.IncomingHttpHeaders;
public readonly url: string;
constructor(incomingMessage: plugins.http.IncomingMessage, url: string) {
this.incomingMessage = incomingMessage;
this.url = url;
this.status = incomingMessage.statusCode || 0;
this.statusText = incomingMessage.statusMessage || '';
this.ok = this.status >= 200 && this.status < 300;
this.headers = incomingMessage.headers;
}
/**
* Ensures the body can only be consumed once
*/
private ensureNotConsumed(): void {
if (this.consumed) {
throw new Error('Body has already been consumed');
}
this.consumed = true;
}
/**
* Collects the body as a buffer
*/
private async collectBody(): Promise<Buffer> {
this.ensureNotConsumed();
if (this.bodyBufferPromise) {
return this.bodyBufferPromise;
}
this.bodyBufferPromise = new Promise<Buffer>((resolve, reject) => {
const chunks: Buffer[] = [];
this.incomingMessage.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
this.incomingMessage.on('end', () => {
resolve(Buffer.concat(chunks));
});
this.incomingMessage.on('error', reject);
});
return this.bodyBufferPromise;
}
/**
* Parse response as JSON
*/
async json(): Promise<T> {
const buffer = await this.collectBody();
const text = buffer.toString('utf-8');
try {
return JSON.parse(text);
} catch (error) {
throw new Error(`Failed to parse JSON: ${error.message}`);
}
}
/**
* Get response as text
*/
async text(): Promise<string> {
const buffer = await this.collectBody();
return buffer.toString('utf-8');
}
/**
* Get response as ArrayBuffer
*/
async arrayBuffer(): Promise<ArrayBuffer> {
const buffer = await this.collectBody();
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
}
/**
* Get response as a readable stream
*/
stream(): NodeJS.ReadableStream {
this.ensureNotConsumed();
return this.incomingMessage;
}
/**
* Get the raw IncomingMessage (for legacy compatibility)
*/
raw(): plugins.http.IncomingMessage {
return this.incomingMessage;
}
}

67
ts/core/types.ts Normal file
View File

@@ -0,0 +1,67 @@
import * as plugins from './plugins.js';
/**
* Core request options extending Node.js RequestOptions
*/
export interface ICoreRequestOptions extends plugins.https.RequestOptions {
keepAlive?: boolean;
requestBody?: any;
queryParams?: { [key: string]: string };
hardDataCuttingTimeout?: number;
}
/**
* HTTP Methods supported
*/
export type THttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
/**
* Response types supported
*/
export type ResponseType = 'json' | 'text' | 'binary' | 'stream';
/**
* Extended IncomingMessage with body property (legacy compatibility)
*/
export interface IExtendedIncomingMessage<T = any> extends plugins.http.IncomingMessage {
body: T;
}
/**
* Form field data for multipart/form-data requests
*/
export interface IFormField {
name: string;
value: string | Buffer;
filename?: string;
contentType?: string;
}
/**
* URL encoded form field
*/
export interface IUrlEncodedField {
key: string;
value: string;
}
/**
* Core response object that provides fetch-like API
*/
export interface ICoreResponse<T = any> {
// Properties
ok: boolean;
status: number;
statusText: string;
headers: plugins.http.IncomingHttpHeaders;
url: string;
// Methods
json(): Promise<T>;
text(): Promise<string>;
arrayBuffer(): Promise<ArrayBuffer>;
stream(): NodeJS.ReadableStream;
// Legacy compatibility
raw(): plugins.http.IncomingMessage;
}

View File

@@ -1,7 +1,12 @@
export { request } from './smartrequest.request.js';
export type { IExtendedIncomingMessage } from './smartrequest.request.js';
export type { ISmartRequestOptions } from './smartrequest.interfaces.js';
// Legacy API exports (for backward compatibility)
export * from './legacy/index.js';
export * from './smartrequest.jsonrest.js';
export * from './smartrequest.binaryrest.js';
export * from './smartrequest.formdata.js';
// Modern API exports
export * from './modern/index.js';
// Core exports for advanced usage
export { SmartResponse, type ICoreRequestOptions, type ICoreResponse } from './core/index.js';
// Default export for easier importing
import { SmartRequestClient } from './modern/smartrequestclient.js';
export default SmartRequestClient;

242
ts/legacy/adapter.ts Normal file
View File

@@ -0,0 +1,242 @@
/**
* Legacy adapter that provides backward compatibility
* Maps legacy API to the new core module
*/
import * as core from '../core/index.js';
import * as plugins from '../core/plugins.js';
const smartpromise = plugins.smartpromise;
// Re-export types for backward compatibility
export { type IExtendedIncomingMessage } from '../core/types.js';
export interface ISmartRequestOptions extends core.ICoreRequestOptions {
autoJsonParse?: boolean;
responseType?: 'json' | 'text' | 'binary' | 'stream';
}
// Re-export interface for form fields
export interface IFormField {
name: string;
type: 'string' | 'filePath' | 'Buffer';
payload: string | Buffer;
fileName?: string;
contentType?: string;
}
/**
* Helper function to convert stream to IExtendedIncomingMessage for legacy compatibility
*/
async function streamToExtendedMessage(
stream: plugins.http.IncomingMessage,
autoJsonParse = true
): Promise<core.IExtendedIncomingMessage> {
const done = smartpromise.defer<core.IExtendedIncomingMessage>();
const chunks: Buffer[] = [];
stream.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
stream.on('end', () => {
const buffer = Buffer.concat(chunks);
const extendedMessage = stream as core.IExtendedIncomingMessage;
if (autoJsonParse) {
const text = buffer.toString('utf-8');
try {
extendedMessage.body = JSON.parse(text);
} catch (err) {
extendedMessage.body = text;
}
} else {
extendedMessage.body = buffer;
}
done.resolve(extendedMessage);
});
stream.on('error', (err) => {
done.reject(err);
});
return done.promise;
}
/**
* Legacy request function that returns IExtendedIncomingMessage
*/
export async function request(
urlArg: string,
optionsArg: ISmartRequestOptions = {},
responseStreamArg = false,
requestDataFunc?: (req: plugins.http.ClientRequest) => void
): Promise<core.IExtendedIncomingMessage> {
const stream = await core.coreRequest(urlArg, optionsArg, requestDataFunc);
if (responseStreamArg) {
// For stream responses, just cast and return
return stream as core.IExtendedIncomingMessage;
}
// Convert stream to IExtendedIncomingMessage
const autoJsonParse = optionsArg.autoJsonParse !== false;
return streamToExtendedMessage(stream, autoJsonParse);
}
/**
* Safe GET request
*/
export async function safeGet(urlArg: string): Promise<core.IExtendedIncomingMessage | null> {
const agentToUse = urlArg.startsWith('http://')
? new plugins.http.Agent()
: new plugins.https.Agent();
try {
const response = await request(urlArg, {
method: 'GET',
agent: agentToUse,
timeout: 5000,
hardDataCuttingTimeout: 5000,
autoJsonParse: false,
});
return response;
} catch (err) {
console.error(err);
return null;
}
}
/**
* GET JSON request
*/
export async function getJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) {
optionsArg.method = 'GET';
return request(urlArg, optionsArg);
}
/**
* POST JSON request
*/
export async function postJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) {
optionsArg.method = 'POST';
if (
typeof optionsArg.requestBody === 'object' &&
(!optionsArg.headers || !optionsArg.headers['Content-Type'])
) {
// make sure headers exist
if (!optionsArg.headers) {
optionsArg.headers = {};
}
// assign the right Content-Type, leaving all other headers in place
optionsArg.headers = {
...optionsArg.headers,
'Content-Type': 'application/json',
};
}
return request(urlArg, optionsArg);
}
/**
* PUT JSON request
*/
export async function putJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) {
optionsArg.method = 'PUT';
return request(urlArg, optionsArg);
}
/**
* DELETE JSON request
*/
export async function delJson(urlArg: string, optionsArg: ISmartRequestOptions = {}) {
optionsArg.method = 'DELETE';
return request(urlArg, optionsArg);
}
/**
* GET binary data
*/
export async function getBinary(urlArg: string, optionsArg: ISmartRequestOptions = {}) {
optionsArg = {
...optionsArg,
autoJsonParse: false,
responseType: 'binary'
};
return request(urlArg, optionsArg);
}
/**
* POST form data
*/
export async function postFormData(urlArg: string, formFields: IFormField[], optionsArg: ISmartRequestOptions = {}) {
const form = new plugins.formData();
for (const formField of formFields) {
if (formField.type === 'filePath') {
const fileData = plugins.fs.readFileSync(
plugins.path.isAbsolute(formField.payload as string)
? formField.payload as string
: plugins.path.join(process.cwd(), formField.payload as string)
);
form.append(formField.name, fileData, {
filename: formField.fileName || plugins.path.basename(formField.payload as string),
contentType: formField.contentType
});
} else if (formField.type === 'Buffer') {
form.append(formField.name, formField.payload, {
filename: formField.fileName,
contentType: formField.contentType
});
} else {
form.append(formField.name, formField.payload);
}
}
optionsArg.method = 'POST';
optionsArg.requestBody = form;
if (!optionsArg.headers) {
optionsArg.headers = {};
}
optionsArg.headers = {
...optionsArg.headers,
...form.getHeaders()
};
return request(urlArg, optionsArg);
}
/**
* POST URL encoded form data
*/
export async function postFormDataUrlEncoded(
urlArg: string,
formFields: { key: string; content: string }[],
optionsArg: ISmartRequestOptions = {}
) {
optionsArg.method = 'POST';
if (!optionsArg.headers) {
optionsArg.headers = {};
}
optionsArg.headers['Content-Type'] = 'application/x-www-form-urlencoded';
const urlEncodedBody = formFields
.map(field => `${encodeURIComponent(field.key)}=${encodeURIComponent(field.content)}`)
.join('&');
optionsArg.requestBody = urlEncodedBody;
return request(urlArg, optionsArg);
}
/**
* GET stream
*/
export async function getStream(
urlArg: string,
optionsArg: ISmartRequestOptions = {}
): Promise<plugins.http.IncomingMessage> {
optionsArg.method = 'GET';
const response = await request(urlArg, optionsArg, true);
return response;
}

2
ts/legacy/index.ts Normal file
View File

@@ -0,0 +1,2 @@
// Export everything from the legacy adapter
export * from './adapter.js';

View File

@@ -0,0 +1,175 @@
import { type SmartResponse } from '../../core/index.js';
import { type TPaginationConfig, PaginationStrategy, type TPaginatedResponse } from '../types/pagination.js';
/**
* Creates a paginated response from a regular response
*/
export async function createPaginatedResponse<T>(
response: SmartResponse<any>,
paginationConfig: TPaginationConfig,
queryParams: Record<string, string>,
fetchNextPage: (params: Record<string, string>) => Promise<TPaginatedResponse<T>>
): Promise<TPaginatedResponse<T>> {
// Parse response body first
const body = await response.json();
// Default to response.body for items if response is JSON
let items: T[] = Array.isArray(body)
? body
: (body?.items || body?.data || body?.results || []);
let hasNextPage = false;
let nextPageParams: Record<string, string> = {};
// Determine if there's a next page based on pagination strategy
switch (paginationConfig.strategy) {
case PaginationStrategy.OFFSET: {
const config = paginationConfig;
const currentPage = parseInt(queryParams[config.pageParam || 'page'] || String(config.startPage || 1));
const limit = parseInt(queryParams[config.limitParam || 'limit'] || String(config.pageSize || 20));
const total = getValueByPath(body, config.totalPath || 'total') || 0;
hasNextPage = currentPage * limit < total;
if (hasNextPage) {
nextPageParams = {
...queryParams,
[config.pageParam || 'page']: String(currentPage + 1)
};
}
break;
}
case PaginationStrategy.CURSOR: {
const config = paginationConfig;
const nextCursor = getValueByPath(body, config.cursorPath || 'nextCursor');
const hasMore = getValueByPath(body, config.hasMorePath || 'hasMore');
hasNextPage = !!nextCursor || !!hasMore;
if (hasNextPage && nextCursor) {
nextPageParams = {
...queryParams,
[config.cursorParam || 'cursor']: nextCursor
};
}
break;
}
case PaginationStrategy.LINK_HEADER: {
const linkHeader = response.headers['link'] || '';
// Handle both string and string[] types for the link header
const headerValue = Array.isArray(linkHeader) ? linkHeader[0] : linkHeader;
const links = parseLinkHeader(headerValue);
hasNextPage = !!links.next;
if (hasNextPage && links.next) {
// Extract query parameters from next link URL
const url = new URL(links.next);
nextPageParams = {};
url.searchParams.forEach((value, key) => {
nextPageParams[key] = value;
});
}
break;
}
case PaginationStrategy.CUSTOM: {
const config = paginationConfig;
hasNextPage = config.hasNextPage(response);
if (hasNextPage) {
nextPageParams = config.getNextPageParams(response, queryParams);
}
break;
}
}
// Create a function to fetch the next page
const getNextPage = async (): Promise<TPaginatedResponse<T>> => {
if (!hasNextPage) {
throw new Error('No more pages available');
}
return fetchNextPage(nextPageParams);
};
// Create a function to fetch all remaining pages
const getAllPages = async (): Promise<T[]> => {
const allItems = [...items];
let currentPage: TPaginatedResponse<T> = { items, hasNextPage, getNextPage, getAllPages, response };
while (currentPage.hasNextPage) {
try {
currentPage = await currentPage.getNextPage();
allItems.push(...currentPage.items);
} catch (error) {
break;
}
}
return allItems;
};
return {
items,
hasNextPage,
getNextPage,
getAllPages,
response
};
}
/**
* Parse Link header for pagination
* Link: <https://api.example.com/users?page=2>; rel="next", <https://api.example.com/users?page=5>; rel="last"
*/
export function parseLinkHeader(header: string): Record<string, string> {
const links: Record<string, string> = {};
if (!header) {
return links;
}
// Split parts by comma
const parts = header.split(',');
// Parse each part into a name:value pair
for (const part of parts) {
const section = part.split(';');
if (section.length < 2) {
continue;
}
const url = section[0].replace(/<(.*)>/, '$1').trim();
const name = section[1].replace(/rel="(.*)"/, '$1').trim();
links[name] = url;
}
return links;
}
/**
* Get a nested value from an object using dot notation path
* e.g., getValueByPath(obj, "data.pagination.nextCursor")
*/
export function getValueByPath(obj: any, path?: string): any {
if (!path || !obj) {
return undefined;
}
const keys = path.split('.');
let current = obj;
for (const key of keys) {
if (current === null || current === undefined || typeof current !== 'object') {
return undefined;
}
current = current[key];
}
return current;
}

48
ts/modern/index.ts Normal file
View File

@@ -0,0 +1,48 @@
// Export the main client
export { SmartRequestClient } from './smartrequestclient.js';
// Export response type from core
export { SmartResponse } from '../core/index.js';
// Export types
export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig } from './types/common.js';
export {
PaginationStrategy,
type TPaginationConfig as PaginationConfig,
type OffsetPaginationConfig,
type CursorPaginationConfig,
type LinkPaginationConfig,
type CustomPaginationConfig,
type TPaginatedResponse as PaginatedResponse
} from './types/pagination.js';
// Convenience factory functions
import { SmartRequestClient } from './smartrequestclient.js';
/**
* Create a client pre-configured for JSON requests
*/
export function createJsonClient<T = any>() {
return SmartRequestClient.create<T>();
}
/**
* Create a client pre-configured for form data requests
*/
export function createFormClient<T = any>() {
return SmartRequestClient.create<T>();
}
/**
* Create a client pre-configured for binary data
*/
export function createBinaryClient<T = any>() {
return SmartRequestClient.create<T>().accept('binary');
}
/**
* Create a client pre-configured for streaming
*/
export function createStreamClient() {
return SmartRequestClient.create().accept('stream');
}

View File

@@ -0,0 +1,329 @@
import { request, SmartResponse, type ICoreRequestOptions } from '../core/index.js';
import * as plugins from '../core/plugins.js';
import type { HttpMethod, ResponseType, FormField } from './types/common.js';
import {
type TPaginationConfig,
PaginationStrategy,
type OffsetPaginationConfig,
type CursorPaginationConfig,
type CustomPaginationConfig,
type TPaginatedResponse
} from './types/pagination.js';
import { createPaginatedResponse } from './features/pagination.js';
/**
* Modern fluent client for making HTTP requests
*/
export class SmartRequestClient<T = any> {
private _url: string;
private _options: ICoreRequestOptions = {};
private _retries: number = 0;
private _queryParams: Record<string, string> = {};
private _paginationConfig?: TPaginationConfig;
/**
* Create a new SmartRequestClient instance
*/
static create<T = any>(): SmartRequestClient<T> {
return new SmartRequestClient<T>();
}
/**
* Set the URL for the request
*/
url(url: string): this {
this._url = url;
return this;
}
/**
* Set the HTTP method
*/
method(method: HttpMethod): this {
this._options.method = method;
return this;
}
/**
* Set JSON body for the request
*/
json(data: any): this {
if (!this._options.headers) {
this._options.headers = {};
}
this._options.headers['Content-Type'] = 'application/json';
this._options.requestBody = data;
return this;
}
/**
* Set form data for the request
*/
formData(data: FormField[]): this {
const form = new plugins.formData();
for (const item of data) {
if (Buffer.isBuffer(item.value)) {
form.append(item.name, item.value, {
filename: item.filename || 'file',
contentType: item.contentType || 'application/octet-stream'
});
} else {
form.append(item.name, item.value);
}
}
if (!this._options.headers) {
this._options.headers = {};
}
this._options.headers = {
...this._options.headers,
...form.getHeaders()
};
this._options.requestBody = form;
return this;
}
/**
* Set request timeout in milliseconds
*/
timeout(ms: number): this {
this._options.timeout = ms;
this._options.hardDataCuttingTimeout = ms;
return this;
}
/**
* Set number of retry attempts
*/
retry(count: number): this {
this._retries = count;
return this;
}
/**
* Set HTTP headers
*/
headers(headers: Record<string, string>): this {
if (!this._options.headers) {
this._options.headers = {};
}
this._options.headers = {
...this._options.headers,
...headers
};
return this;
}
/**
* Set a single HTTP header
*/
header(name: string, value: string): this {
if (!this._options.headers) {
this._options.headers = {};
}
this._options.headers[name] = value;
return this;
}
/**
* Set query parameters
*/
query(params: Record<string, string>): this {
this._queryParams = {
...this._queryParams,
...params
};
return this;
}
/**
* Set the Accept header to indicate what content type is expected
*/
accept(type: ResponseType): this {
// Map response types to Accept header values
const acceptHeaders: Record<ResponseType, string> = {
'json': 'application/json',
'text': 'text/plain',
'binary': 'application/octet-stream',
'stream': '*/*'
};
return this.header('Accept', acceptHeaders[type]);
}
/**
* Configure pagination for requests
*/
pagination(config: TPaginationConfig): this {
this._paginationConfig = config;
return this;
}
/**
* Configure offset-based pagination (page & limit)
*/
withOffsetPagination(config: Omit<OffsetPaginationConfig, 'strategy'> = {}): this {
this._paginationConfig = {
strategy: PaginationStrategy.OFFSET,
pageParam: config.pageParam || 'page',
limitParam: config.limitParam || 'limit',
startPage: config.startPage || 1,
pageSize: config.pageSize || 20,
totalPath: config.totalPath || 'total'
};
// Add initial pagination parameters
this.query({
[this._paginationConfig.pageParam]: String(this._paginationConfig.startPage),
[this._paginationConfig.limitParam]: String(this._paginationConfig.pageSize)
});
return this;
}
/**
* Configure cursor-based pagination
*/
withCursorPagination(config: Omit<CursorPaginationConfig, 'strategy'> = {}): this {
this._paginationConfig = {
strategy: PaginationStrategy.CURSOR,
cursorParam: config.cursorParam || 'cursor',
cursorPath: config.cursorPath || 'nextCursor',
hasMorePath: config.hasMorePath || 'hasMore'
};
return this;
}
/**
* Configure Link header-based pagination
*/
withLinkPagination(): this {
this._paginationConfig = {
strategy: PaginationStrategy.LINK_HEADER
};
return this;
}
/**
* Configure custom pagination
*/
withCustomPagination(config: Omit<CustomPaginationConfig, 'strategy'>): this {
this._paginationConfig = {
strategy: PaginationStrategy.CUSTOM,
hasNextPage: config.hasNextPage,
getNextPageParams: config.getNextPageParams
};
return this;
}
/**
* Make a GET request
*/
async get<R = T>(): Promise<SmartResponse<R>> {
return this.execute<R>('GET');
}
/**
* Make a POST request
*/
async post<R = T>(): Promise<SmartResponse<R>> {
return this.execute<R>('POST');
}
/**
* Make a PUT request
*/
async put<R = T>(): Promise<SmartResponse<R>> {
return this.execute<R>('PUT');
}
/**
* Make a DELETE request
*/
async delete<R = T>(): Promise<SmartResponse<R>> {
return this.execute<R>('DELETE');
}
/**
* Make a PATCH request
*/
async patch<R = T>(): Promise<SmartResponse<R>> {
return this.execute<R>('PATCH');
}
/**
* Get paginated response
*/
async getPaginated<ItemType = T>(): Promise<TPaginatedResponse<ItemType>> {
if (!this._paginationConfig) {
throw new Error('Pagination not configured. Call one of the pagination methods first.');
}
// Default to GET if no method specified
if (!this._options.method) {
this._options.method = 'GET';
}
const response = await this.execute();
return await createPaginatedResponse<ItemType>(
response,
this._paginationConfig,
this._queryParams,
(nextPageParams) => {
// Create a new client with the same configuration but updated query params
const nextClient = new SmartRequestClient<ItemType>();
Object.assign(nextClient, this);
nextClient._queryParams = nextPageParams;
return nextClient.getPaginated<ItemType>();
}
);
}
/**
* Get all pages at once (use with caution for large datasets)
*/
async getAllPages<ItemType = T>(): Promise<ItemType[]> {
const firstPage = await this.getPaginated<ItemType>();
return firstPage.getAllPages();
}
/**
* Execute the HTTP request
*/
private async execute<R = T>(method?: HttpMethod): Promise<SmartResponse<R>> {
if (method) {
this._options.method = method;
}
this._options.queryParams = this._queryParams;
// Handle retry logic
let lastError: Error;
for (let attempt = 0; attempt <= this._retries; attempt++) {
try {
const response = await request(this._url, this._options);
return response as SmartResponse<R>;
} catch (error) {
lastError = error as Error;
// If this is the last attempt, throw the error
if (attempt === this._retries) {
throw lastError;
}
// Otherwise, wait before retrying
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
// This should never be reached due to the throw in the loop above
throw lastError;
}
}

49
ts/modern/types/common.ts Normal file
View File

@@ -0,0 +1,49 @@
/**
* HTTP Methods supported by the client
*/
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
/**
* Response types supported by the client
*/
export type ResponseType = 'json' | 'text' | 'binary' | 'stream';
/**
* Form field data for multipart/form-data requests
*/
export interface FormField {
name: string;
value: string | Buffer;
filename?: string;
contentType?: string;
}
/**
* URL encoded form field
*/
export interface UrlEncodedField {
key: string;
value: string;
}
/**
* Retry configuration
*/
export interface RetryConfig {
attempts: number; // Number of retry attempts
initialDelay?: number; // Initial delay in ms
maxDelay?: number; // Maximum delay in ms
factor?: number; // Backoff factor
statusCodes?: number[]; // Status codes to retry on
shouldRetry?: (error: Error, attemptCount: number) => boolean;
}
/**
* Timeout configuration
*/
export interface TimeoutConfig {
request?: number; // Overall request timeout in ms
connection?: number; // Connection timeout in ms
socket?: number; // Socket idle timeout in ms
response?: number; // Response timeout in ms
}

View File

@@ -0,0 +1,66 @@
import { type SmartResponse } from '../../core/index.js';
/**
* Pagination strategy options
*/
export enum PaginationStrategy {
OFFSET = 'offset', // Uses page & limit parameters
CURSOR = 'cursor', // Uses a cursor/token for next page
LINK_HEADER = 'link', // Uses Link headers
CUSTOM = 'custom' // Uses a custom pagination handler
}
/**
* Configuration for offset-based pagination
*/
export interface OffsetPaginationConfig {
strategy: PaginationStrategy.OFFSET;
pageParam?: string; // Parameter name for page number (default: "page")
limitParam?: string; // Parameter name for page size (default: "limit")
startPage?: number; // Starting page number (default: 1)
pageSize?: number; // Number of items per page (default: 20)
totalPath?: string; // JSON path to total item count (default: "total")
}
/**
* Configuration for cursor-based pagination
*/
export interface CursorPaginationConfig {
strategy: PaginationStrategy.CURSOR;
cursorParam?: string; // Parameter name for cursor (default: "cursor")
cursorPath?: string; // JSON path to next cursor (default: "nextCursor")
hasMorePath?: string; // JSON path to check if more items exist (default: "hasMore")
}
/**
* Configuration for Link header-based pagination
*/
export interface LinkPaginationConfig {
strategy: PaginationStrategy.LINK_HEADER;
// No additional config needed, uses standard Link header format
}
/**
* Configuration for custom pagination
*/
export interface CustomPaginationConfig {
strategy: PaginationStrategy.CUSTOM;
hasNextPage: (response: SmartResponse<any>) => boolean;
getNextPageParams: (response: SmartResponse<any>, currentParams: Record<string, string>) => Record<string, string>;
}
/**
* Union type of all pagination configurations
*/
export type TPaginationConfig = OffsetPaginationConfig | CursorPaginationConfig | LinkPaginationConfig | CustomPaginationConfig;
/**
* Interface for a paginated response
*/
export interface TPaginatedResponse<T> {
items: T[]; // Current page items
hasNextPage: boolean; // Whether there are more pages
getNextPage: () => Promise<TPaginatedResponse<T>>; // Function to get the next page
getAllPages: () => Promise<T[]>; // Function to get all remaining pages and combine
response: SmartResponse<any>; // Original response
}

View File

@@ -1,33 +0,0 @@
// this file implements methods to get and post binary data.
import * as interfaces from './smartrequest.interfaces.js';
import { request } from './smartrequest.request.js';
import * as plugins from './smartrequest.plugins.js';
export const getBinary = async (
domainArg: string,
optionsArg: interfaces.ISmartRequestOptions = {}
) => {
optionsArg = {
...optionsArg,
autoJsonParse: false,
};
const done = plugins.smartpromise.defer();
const response = await request(domainArg, optionsArg, true);
const data: Array<Buffer> = [];
response
.on('data', function (chunk: Buffer) {
data.push(chunk);
})
.on('end', function () {
//at this point data is an array of Buffers
//so Buffer.concat() can make us a new Buffer
//of all of them together
const buffer = Buffer.concat(data);
response.body = buffer;
done.resolve();
});
await done.promise;
return response;
};

View File

@@ -1,99 +0,0 @@
import * as plugins from './smartrequest.plugins.js';
import * as interfaces from './smartrequest.interfaces.js';
import { request } from './smartrequest.request.js';
/**
* the interfae for FormFieldData
*/
export interface IFormField {
name: string;
type: 'string' | 'filePath' | 'Buffer';
payload: string | Buffer;
fileName?: string;
contentType?: string;
}
const appendFormField = async (formDataArg: plugins.formData, formDataField: IFormField) => {
switch (formDataField.type) {
case 'string':
formDataArg.append(formDataField.name, formDataField.payload);
break;
case 'filePath':
if (typeof formDataField.payload !== 'string') {
throw new Error(
`Payload for key ${
formDataField.name
} must be of type string. Got ${typeof formDataField.payload} instead.`
);
}
const fileData = plugins.fs.readFileSync(
plugins.path.join(process.cwd(), formDataField.payload)
);
formDataArg.append('file', fileData, {
filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf',
contentType: 'application/pdf',
});
break;
case 'Buffer':
formDataArg.append(formDataField.name, formDataField.payload, {
filename: formDataField.fileName ? formDataField.fileName : 'upload.pdf',
contentType: formDataField.contentType ? formDataField.contentType : 'application/pdf',
});
break;
}
};
export const postFormData = async (
urlArg: string,
optionsArg: interfaces.ISmartRequestOptions = {},
payloadArg: IFormField[]
) => {
const form = new plugins.formData();
for (const formField of payloadArg) {
await appendFormField(form, formField);
}
const requestOptions = {
...optionsArg,
method: 'POST',
headers: {
...optionsArg.headers,
...form.getHeaders(),
},
requestBody: form,
};
// lets fire the actual request for sending the formdata
const response = await request(urlArg, requestOptions);
return response;
};
export const postFormDataUrlEncoded = async (
urlArg: string,
optionsArg: interfaces.ISmartRequestOptions = {},
payloadArg: { key: string; content: string }[]
) => {
let resultString = '';
for (const keyContentPair of payloadArg) {
if (resultString) {
resultString += '&';
}
resultString += `${encodeURIComponent(keyContentPair.key)}=${encodeURIComponent(
keyContentPair.content
)}`;
}
const requestOptions: interfaces.ISmartRequestOptions = {
...optionsArg,
method: 'POST',
headers: {
...optionsArg.headers,
'content-type': 'application/x-www-form-urlencoded',
},
requestBody: resultString,
};
// lets fire the actual request for sending the formdata
const response = await request(urlArg, requestOptions);
return response;
};

View File

@@ -1,9 +0,0 @@
import * as plugins from './smartrequest.plugins.js';
import * as https from 'https';
export interface ISmartRequestOptions extends https.RequestOptions {
keepAlive?: boolean;
requestBody?: any;
autoJsonParse?: boolean;
queryParams?: { [key: string]: string };
}

View File

@@ -1,63 +0,0 @@
// This file implements methods to get and post JSON in a simple manner.
import * as interfaces from './smartrequest.interfaces.js';
import { request } from './smartrequest.request.js';
/**
* gets Json and puts the right headers + handles response aggregation
* @param domainArg
* @param optionsArg
*/
export const getJson = async (
domainArg: string,
optionsArg: interfaces.ISmartRequestOptions = {}
) => {
optionsArg.method = 'GET';
optionsArg.headers = {
...optionsArg.headers,
};
let response = await request(domainArg, optionsArg);
return response;
};
export const postJson = async (
domainArg: string,
optionsArg: interfaces.ISmartRequestOptions = {}
) => {
optionsArg.method = 'POST';
if (
typeof optionsArg.requestBody === 'object' &&
(!optionsArg.headers || !optionsArg.headers['Content-Type'])
) {
// make sure headers exist
if (!optionsArg.headers) {
optionsArg.headers = {};
}
// assign the right Content-Type, leaving all other headers in place
optionsArg.headers = {
...optionsArg.headers,
'Content-Type': 'application/json',
};
}
let response = await request(domainArg, optionsArg);
return response;
};
export const putJson = async (
domainArg: string,
optionsArg: interfaces.ISmartRequestOptions = {}
) => {
optionsArg.method = 'PUT';
let response = await request(domainArg, optionsArg);
return response;
};
export const delJson = async (
domainArg: string,
optionsArg: interfaces.ISmartRequestOptions = {}
) => {
optionsArg.method = 'DELETE';
let response = await request(domainArg, optionsArg);
return response;
};

View File

@@ -1,194 +0,0 @@
import * as plugins from './smartrequest.plugins.js';
import * as interfaces from './smartrequest.interfaces.js';
import { IncomingMessage } from 'http';
export interface IExtendedIncomingMessage extends IncomingMessage {
body: any;
}
const buildUtf8Response = (
incomingMessageArg: IncomingMessage,
autoJsonParse = true
): Promise<IExtendedIncomingMessage> => {
const done = plugins.smartpromise.defer<IExtendedIncomingMessage>();
// Continuously update stream with data
let body = '';
incomingMessageArg.on('data', (chunkArg) => {
body += chunkArg;
});
incomingMessageArg.on('end', () => {
if (autoJsonParse) {
try {
(incomingMessageArg as IExtendedIncomingMessage).body = JSON.parse(body);
} catch (err) {
(incomingMessageArg as IExtendedIncomingMessage).body = body;
}
} else {
(incomingMessageArg as IExtendedIncomingMessage).body = body;
}
done.resolve(incomingMessageArg as IExtendedIncomingMessage);
});
return done.promise;
};
/**
* determine wether a url is a unix sock
* @param urlArg
*/
const testForUnixSock = (urlArg: string): boolean => {
const unixRegex = /^(http:\/\/|https:\/\/|)unix:/;
return unixRegex.test(urlArg);
};
/**
* determine socketPath and path for unixsock
*/
const parseSocketPathAndRoute = (stringToParseArg: string) => {
const parseRegex = /(.*):(.*)/;
const result = parseRegex.exec(stringToParseArg);
return {
socketPath: result[1],
path: result[2],
};
};
/**
* a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
*/
const httpAgent = new plugins.agentkeepalive({
keepAlive: true,
maxFreeSockets: 10,
maxSockets: 100,
maxTotalSockets: 1000,
timeout: 60000,
});
/**
* a custom http agent to make sure we can set custom keepAlive options for speedy subsequent calls
*/
const httpAgentKeepAliveFalse = new plugins.agentkeepalive({
keepAlive: false,
timeout: 60000
});
/**
* a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
*/
const httpsAgent = new plugins.agentkeepalive.HttpsAgent({
keepAlive: true,
maxFreeSockets: 10,
maxSockets: 100,
maxTotalSockets: 1000,
timeout: 60000
});
/**
* a custom https agent to make sure we can set custom keepAlive options for speedy subsequent calls
*/
const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({
keepAlive: false,
timeout: 60000
});
export let request = async (
urlArg: string,
optionsArg: interfaces.ISmartRequestOptions = {},
responseStreamArg: boolean = false,
requestDataFunc: (req: plugins.http.ClientRequest) => void = null
): Promise<IExtendedIncomingMessage> => {
const done = plugins.smartpromise.defer<any>();
// merge options
const defaultOptions: interfaces.ISmartRequestOptions = {
// agent: agent,
autoJsonParse: true,
keepAlive: true,
};
optionsArg = {
...defaultOptions,
...optionsArg,
};
// parse url
const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(urlArg, {
searchParams: optionsArg.queryParams || {},
});
optionsArg.hostname = parsedUrl.hostname;
if (parsedUrl.port) {
optionsArg.port = parseInt(parsedUrl.port, 10);
}
optionsArg.path = parsedUrl.path;
optionsArg.queryParams = parsedUrl.searchParams;
// determine if unixsock
if (testForUnixSock(urlArg)) {
const detailedUnixPath = parseSocketPathAndRoute(optionsArg.path);
optionsArg.socketPath = detailedUnixPath.socketPath;
optionsArg.path = detailedUnixPath.path;
}
// TODO: support tcp sockets
// lets determine the request module to use
const requestModule = (() => {
switch (true) {
case parsedUrl.protocol === 'https' && optionsArg.keepAlive:
optionsArg.agent = httpsAgent;
return plugins.https;
case parsedUrl.protocol === 'https' && !optionsArg.keepAlive:
optionsArg.agent = httpsAgentKeepAliveFalse;
return plugins.https;
case parsedUrl.protocol === 'http' && optionsArg.keepAlive:
optionsArg.agent = httpAgent;
return plugins.http;
case parsedUrl.protocol === 'http' && !optionsArg.keepAlive:
optionsArg.agent = httpAgentKeepAliveFalse;
return plugins.http;
}
})() as typeof plugins.https;
if (!requestModule) {
console.error(`The request to ${urlArg} is missing a viable protocol. Must be http or https`);
return;
}
// lets perform the actual request
const requestToFire = requestModule.request(optionsArg, async (response) => {
if (responseStreamArg) {
done.resolve(response);
} else {
const builtResponse = await buildUtf8Response(response, optionsArg.autoJsonParse);
done.resolve(builtResponse);
}
});
// lets write the requestBody
if (optionsArg.requestBody) {
if (optionsArg.requestBody instanceof plugins.formData) {
optionsArg.requestBody.pipe(requestToFire).on('finish', (event: any) => {
requestToFire.end();
});
} else {
if (typeof optionsArg.requestBody !== 'string') {
optionsArg.requestBody = JSON.stringify(optionsArg.requestBody);
}
requestToFire.write(optionsArg.requestBody);
requestToFire.end();
}
} else if (requestDataFunc) {
requestDataFunc(requestToFire);
} else {
requestToFire.end();
}
// lets handle an error
requestToFire.on('error', (e) => {
console.error(e);
});
const result = await done.promise;
return result;
};

View File

@@ -3,8 +3,12 @@
"experimentalDecorators": true,
"useDefineForClassFields": false,
"target": "ES2022",
"module": "ES2022",
"moduleResolution": "nodenext",
"esModuleInterop": true
}
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"verbatimModuleSyntax": true
},
"exclude": [
"dist_*/**/*.d.ts"
]
}