push.rocks/smartrequest
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(client): Fix CI configuration, prevent socket hangs with auto-drain, and apply various client/core TypeScript fixes and test updates

Browse Source
This commit is contained in:
Jürgen Kunz
2025-08-18 00:21:14 +00:00
parent 9b9c8fd618
commit ee750dea58
34 changed files with 2144 additions and 892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartrequest',
version: '2.1.0',
version: '4.2.2',
description: 'A module for modern HTTP/HTTPS requests with support for form data, file uploads, JSON, binary data, streams, and more.'
}

57
ts/client/features/pagination.ts
View File

@@ -1,6 +1,10 @@
import { type CoreResponse } from '../../core/index.js';
import type { ICoreResponse } from '../../core_base/types.js';
import { type TPaginationConfig, PaginationStrategy, type TPaginatedResponse } from '../types/pagination.js';
import {
type TPaginationConfig,
PaginationStrategy,
type TPaginatedResponse,
} from '../types/pagination.js';
/**
* Creates a paginated response from a regular response
@@ -9,15 +13,17 @@ export async function createPaginatedResponse<T>(
response: ICoreResponse<any>,
paginationConfig: TPaginationConfig,
queryParams: Record<string, string>,
fetchNextPage: (params: Record<string, string>) => Promise<TPaginatedResponse<T>>
fetchNextPage: (
params: Record<string, string>,
) => Promise<TPaginatedResponse<T>>,
): Promise<TPaginatedResponse<T>> {
// Parse response body first
const body = await response.json() as any;
const body = (await response.json()) as any;
// Default to response.body for items if response is JSON
let items: T[] = Array.isArray(body)
? body
: (body?.items || body?.data || body?.results || []);
: body?.items || body?.data || body?.results || [];
let hasNextPage = false;
let nextPageParams: Record<string, string> = {};
@@ -26,8 +32,14 @@ export async function createPaginatedResponse<T>(
switch (paginationConfig.strategy) {
case PaginationStrategy.OFFSET: {
const config = paginationConfig;
const currentPage = parseInt(queryParams[config.pageParam || 'page'] || String(config.startPage || 1));
const limit = parseInt(queryParams[config.limitParam || 'limit'] || String(config.pageSize || 20));
const currentPage = parseInt(
queryParams[config.pageParam || 'page'] ||
String(config.startPage || 1),
);
const limit = parseInt(
queryParams[config.limitParam || 'limit'] ||
String(config.pageSize || 20),
);
const total = getValueByPath(body, config.totalPath || 'total') || 0;
hasNextPage = currentPage * limit < total;
@@ -35,7 +47,7 @@ export async function createPaginatedResponse<T>(
if (hasNextPage) {
nextPageParams = {
...queryParams,
[config.pageParam || 'page']: String(currentPage + 1)
[config.pageParam || 'page']: String(currentPage + 1),
};
}
break;
@@ -43,7 +55,10 @@ export async function createPaginatedResponse<T>(
case PaginationStrategy.CURSOR: {
const config = paginationConfig;
const nextCursor = getValueByPath(body, config.cursorPath || 'nextCursor');
const nextCursor = getValueByPath(
body,
config.cursorPath || 'nextCursor',
);
const hasMore = getValueByPath(body, config.hasMorePath || 'hasMore');
hasNextPage = !!nextCursor || !!hasMore;
@@ -51,7 +66,7 @@ export async function createPaginatedResponse<T>(
if (hasNextPage && nextCursor) {
nextPageParams = {
...queryParams,
[config.cursorParam || 'cursor']: nextCursor
[config.cursorParam || 'cursor']: nextCursor,
};
}
break;
@@ -60,7 +75,9 @@ export async function createPaginatedResponse<T>(
case PaginationStrategy.LINK_HEADER: {
const linkHeader = response.headers['link'] || '';
// Handle both string and string[] types for the link header
const headerValue = Array.isArray(linkHeader) ? linkHeader[0] : linkHeader;
const headerValue = Array.isArray(linkHeader)
? linkHeader[0]
: linkHeader;
const links = parseLinkHeader(headerValue);
hasNextPage = !!links.next;
@@ -100,7 +117,13 @@ export async function createPaginatedResponse<T>(
// Create a function to fetch all remaining pages
const getAllPages = async (): Promise<T[]> => {
const allItems = [...items];
let currentPage: TPaginatedResponse<T> = { items, hasNextPage, getNextPage, getAllPages, response };
let currentPage: TPaginatedResponse<T> = {
items,
hasNextPage,
getNextPage,
getAllPages,
response,
};
while (currentPage.hasNextPage) {
try {
@@ -119,7 +142,7 @@ export async function createPaginatedResponse<T>(
hasNextPage,
getNextPage,
getAllPages,
response
response,
};
}
@@ -166,11 +189,15 @@ export function getValueByPath(obj: any, path?: string): any {
let current = obj;
for (const key of keys) {
if (current === null || current === undefined || typeof current !== 'object') {
if (
current === null ||
current === undefined ||
typeof current !== 'object'
) {
return undefined;
}
current = current[key];
}
return current;
}
}

15
ts/client/index.ts
View File

@@ -5,15 +5,22 @@ export { SmartRequest } from './smartrequest.js';
export { CoreResponse } from '../core/index.js';
// Export types
export type { HttpMethod, ResponseType, FormField, RetryConfig, TimeoutConfig, RateLimitConfig } from './types/common.js';
export {
export type {
HttpMethod,
ResponseType,
FormField,
RetryConfig,
TimeoutConfig,
RateLimitConfig,
} from './types/common.js';
export {
PaginationStrategy,
type TPaginationConfig as PaginationConfig,
type OffsetPaginationConfig,
type CursorPaginationConfig,
type LinkPaginationConfig,
type CustomPaginationConfig,
type TPaginatedResponse as PaginatedResponse
type TPaginatedResponse as PaginatedResponse,
} from './types/pagination.js';
// Convenience factory functions
@@ -45,4 +52,4 @@ export function createBinaryClient<T = any>() {
*/
export function createStreamClient() {
return SmartRequest.create().accept('stream');
}
}

4
ts/client/plugins.ts
View File

@@ -1,6 +1,4 @@
// plugins for client module
import FormData from 'form-data';
export {
FormData as formData
};
export { FormData as formData };

97
ts/client/smartrequest.ts
View File

@@ -3,14 +3,19 @@ import type { ICoreResponse } from '../core_base/types.js';
import * as plugins from './plugins.js';
import type { ICoreRequestOptions } from '../core_base/types.js';
import type { HttpMethod, ResponseType, FormField, RateLimitConfig } from './types/common.js';
import type {
HttpMethod,
ResponseType,
FormField,
RateLimitConfig,
} from './types/common.js';
import {
type TPaginationConfig,
PaginationStrategy,
type OffsetPaginationConfig,
type CursorPaginationConfig,
type CustomPaginationConfig,
type TPaginatedResponse
type TPaginatedResponse,
} from './types/pagination.js';
import { createPaginatedResponse } from './features/pagination.js';
@@ -22,21 +27,21 @@ import { createPaginatedResponse } from './features/pagination.js';
function parseRetryAfter(retryAfter: string | string[]): number {
// Handle array of values (take first)
const value = Array.isArray(retryAfter) ? retryAfter[0] : retryAfter;
if (!value) return 0;
// Try to parse as seconds (number)
const seconds = parseInt(value, 10);
if (!isNaN(seconds)) {
return seconds * 1000;
}
// Try to parse as HTTP date
const retryDate = new Date(value);
if (!isNaN(retryDate.getTime())) {
return Math.max(0, retryDate.getTime() - Date.now());
}
return 0;
}
@@ -96,7 +101,7 @@ export class SmartRequest<T = any> {
if (Buffer.isBuffer(item.value)) {
form.append(item.name, item.value, {
filename: item.filename || 'file',
contentType: item.contentType || 'application/octet-stream'
contentType: item.contentType || 'application/octet-stream',
});
} else {
form.append(item.name, item.value);
@@ -109,7 +114,7 @@ export class SmartRequest<T = any> {
this._options.headers = {
...this._options.headers,
...form.getHeaders()
...form.getHeaders(),
};
this._options.requestBody = form;
@@ -143,7 +148,7 @@ export class SmartRequest<T = any> {
maxWaitTime: config?.maxWaitTime ?? 60000,
fallbackDelay: config?.fallbackDelay ?? 1000,
backoffFactor: config?.backoffFactor ?? 2,
onRateLimit: config?.onRateLimit
onRateLimit: config?.onRateLimit,
};
return this;
}
@@ -157,7 +162,7 @@ export class SmartRequest<T = any> {
}
this._options.headers = {
...this._options.headers,
...headers
...headers,
};
return this;
}
@@ -179,7 +184,7 @@ export class SmartRequest<T = any> {
query(params: Record<string, string>): this {
this._queryParams = {
...this._queryParams,
...params
...params,
};
return this;
}
@@ -190,7 +195,7 @@ export class SmartRequest<T = any> {
options(options: Partial<ICoreRequestOptions>): this {
this._options = {
...this._options,
...options
...options,
};
return this;
}
@@ -210,12 +215,12 @@ export class SmartRequest<T = any> {
accept(type: ResponseType): this {
// Map response types to Accept header values
const acceptHeaders: Record<ResponseType, string> = {
'json': 'application/json',
'text': 'text/plain',
'binary': 'application/octet-stream',
'stream': '*/*'
json: 'application/json',
text: 'text/plain',
binary: 'application/octet-stream',
stream: '*/*',
};
return this.header('Accept', acceptHeaders[type]);
}
@@ -230,20 +235,26 @@ export class SmartRequest<T = any> {
/**
* Configure offset-based pagination (page & limit)
*/
withOffsetPagination(config: Omit<OffsetPaginationConfig, 'strategy'> = {}): this {
withOffsetPagination(
config: Omit<OffsetPaginationConfig, 'strategy'> = {},
): this {
this._paginationConfig = {
strategy: PaginationStrategy.OFFSET,
pageParam: config.pageParam || 'page',
limitParam: config.limitParam || 'limit',
startPage: config.startPage || 1,
pageSize: config.pageSize || 20,
totalPath: config.totalPath || 'total'
totalPath: config.totalPath || 'total',
};
// Add initial pagination parameters
this.query({
[this._paginationConfig.pageParam]: String(this._paginationConfig.startPage),
[this._paginationConfig.limitParam]: String(this._paginationConfig.pageSize)
[this._paginationConfig.pageParam]: String(
this._paginationConfig.startPage,
),
[this._paginationConfig.limitParam]: String(
this._paginationConfig.pageSize,
),
});
return this;
@@ -252,12 +263,14 @@ export class SmartRequest<T = any> {
/**
* Configure cursor-based pagination
*/
withCursorPagination(config: Omit<CursorPaginationConfig, 'strategy'> = {}): this {
withCursorPagination(
config: Omit<CursorPaginationConfig, 'strategy'> = {},
): this {
this._paginationConfig = {
strategy: PaginationStrategy.CURSOR,
cursorParam: config.cursorParam || 'cursor',
cursorPath: config.cursorPath || 'nextCursor',
hasMorePath: config.hasMorePath || 'hasMore'
hasMorePath: config.hasMorePath || 'hasMore',
};
return this;
}
@@ -267,7 +280,7 @@ export class SmartRequest<T = any> {
*/
withLinkPagination(): this {
this._paginationConfig = {
strategy: PaginationStrategy.LINK_HEADER
strategy: PaginationStrategy.LINK_HEADER,
};
return this;
}
@@ -279,7 +292,7 @@ export class SmartRequest<T = any> {
this._paginationConfig = {
strategy: PaginationStrategy.CUSTOM,
hasNextPage: config.hasNextPage,
getNextPageParams: config.getNextPageParams
getNextPageParams: config.getNextPageParams,
};
return this;
}
@@ -324,7 +337,9 @@ export class SmartRequest<T = any> {
*/
async getPaginated<ItemType = T>(): Promise<TPaginatedResponse<ItemType>> {
if (!this._paginationConfig) {
throw new Error('Pagination not configured. Call one of the pagination methods first.');
throw new Error(
'Pagination not configured. Call one of the pagination methods first.',
);
}
// Default to GET if no method specified
@@ -345,7 +360,7 @@ export class SmartRequest<T = any> {
nextClient._queryParams = nextPageParams;
return nextClient.getPaginated<ItemType>();
}
},
);
}
@@ -375,8 +390,8 @@ export class SmartRequest<T = any> {
for (let attempt = 0; attempt <= this._retries; attempt++) {
try {
const request = new CoreRequest(this._url, this._options as any);
const response = await request.fire() as ICoreResponse<R>;
const response = (await request.fire()) as ICoreResponse<R>;
// Check for 429 status if rate limit handling is enabled
if (this._rateLimitConfig && response.status === 429) {
if (rateLimitAttempt >= this._rateLimitConfig.maxRetries) {
@@ -385,18 +400,22 @@ export class SmartRequest<T = any> {
}
let waitTime: number;
if (this._rateLimitConfig.respectRetryAfter && response.headers['retry-after']) {
if (
this._rateLimitConfig.respectRetryAfter &&
response.headers['retry-after']
) {
// Parse Retry-After header
waitTime = parseRetryAfter(response.headers['retry-after']);
// Cap wait time to maxWaitTime
waitTime = Math.min(waitTime, this._rateLimitConfig.maxWaitTime);
} else {
// Use exponential backoff
waitTime = Math.min(
this._rateLimitConfig.fallbackDelay * Math.pow(this._rateLimitConfig.backoffFactor, rateLimitAttempt),
this._rateLimitConfig.maxWaitTime
this._rateLimitConfig.fallbackDelay *
Math.pow(this._rateLimitConfig.backoffFactor, rateLimitAttempt),
this._rateLimitConfig.maxWaitTime,
);
}
@@ -406,14 +425,14 @@ export class SmartRequest<T = any> {
}
// Wait before retrying
await new Promise(resolve => setTimeout(resolve, waitTime));
await new Promise((resolve) => setTimeout(resolve, waitTime));
rateLimitAttempt++;
// Decrement attempt to retry this attempt
attempt--;
continue;
}
// Success or non-429 error response
return response;
} catch (error) {
@@ -425,11 +444,11 @@ export class SmartRequest<T = any> {
}
// Otherwise, wait before retrying
await new Promise(resolve => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}
// This should never be reached due to the throw in the loop above
throw lastError;
}
}
}

39
ts/client/types/common.ts
View File

@@ -1,7 +1,14 @@
/**
* HTTP Methods supported by the client
*/
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
export type HttpMethod =
| 'GET'
| 'POST'
| 'PUT'
| 'DELETE'
| 'PATCH'
| 'HEAD'
| 'OPTIONS';
/**
* Response types supported by the client
@@ -30,11 +37,11 @@ export interface UrlEncodedField {
* Retry configuration
*/
export interface RetryConfig {
attempts: number; // Number of retry attempts
initialDelay?: number; // Initial delay in ms
maxDelay?: number; // Maximum delay in ms
factor?: number; // Backoff factor
statusCodes?: number[]; // Status codes to retry on
attempts: number; // Number of retry attempts
initialDelay?: number; // Initial delay in ms
maxDelay?: number; // Maximum delay in ms
factor?: number; // Backoff factor
statusCodes?: number[]; // Status codes to retry on
shouldRetry?: (error: Error, attemptCount: number) => boolean;
}
@@ -42,20 +49,20 @@ export interface RetryConfig {
* Timeout configuration
*/
export interface TimeoutConfig {
request?: number; // Overall request timeout in ms
connection?: number; // Connection timeout in ms
socket?: number; // Socket idle timeout in ms
response?: number; // Response timeout in ms
request?: number; // Overall request timeout in ms
connection?: number; // Connection timeout in ms
socket?: number; // Socket idle timeout in ms
response?: number; // Response timeout in ms
}
/**
* Rate limit configuration for handling 429 responses
*/
export interface RateLimitConfig {
maxRetries?: number; // Maximum number of retries (default: 3)
respectRetryAfter?: boolean; // Respect Retry-After header (default: true)
maxWaitTime?: number; // Max wait time in ms (default: 60000)
fallbackDelay?: number; // Delay when no Retry-After header (default: 1000)
backoffFactor?: number; // Exponential backoff factor (default: 2)
maxRetries?: number; // Maximum number of retries (default: 3)
respectRetryAfter?: boolean; // Respect Retry-After header (default: true)
maxWaitTime?: number; // Max wait time in ms (default: 60000)
fallbackDelay?: number; // Delay when no Retry-After header (default: 1000)
backoffFactor?: number; // Exponential backoff factor (default: 2)
onRateLimit?: (attempt: number, waitTime: number) => void; // Callback for rate limit events
}
}

47
ts/client/types/pagination.ts
View File

@@ -5,10 +5,10 @@ import type { ICoreResponse } from '../../core_base/types.js';
* Pagination strategy options
*/
export enum PaginationStrategy {
OFFSET = 'offset', // Uses page & limit parameters
CURSOR = 'cursor', // Uses a cursor/token for next page
LINK_HEADER = 'link', // Uses Link headers
CUSTOM = 'custom' // Uses a custom pagination handler
OFFSET = 'offset', // Uses page & limit parameters
CURSOR = 'cursor', // Uses a cursor/token for next page
LINK_HEADER = 'link', // Uses Link headers
CUSTOM = 'custom', // Uses a custom pagination handler
}
/**
@@ -16,11 +16,11 @@ export enum PaginationStrategy {
*/
export interface OffsetPaginationConfig {
strategy: PaginationStrategy.OFFSET;
pageParam?: string; // Parameter name for page number (default: "page")
limitParam?: string; // Parameter name for page size (default: "limit")
startPage?: number; // Starting page number (default: 1)
pageSize?: number; // Number of items per page (default: 20)
totalPath?: string; // JSON path to total item count (default: "total")
pageParam?: string; // Parameter name for page number (default: "page")
limitParam?: string; // Parameter name for page size (default: "limit")
startPage?: number; // Starting page number (default: 1)
pageSize?: number; // Number of items per page (default: 20)
totalPath?: string; // JSON path to total item count (default: "total")
}
/**
@@ -28,9 +28,9 @@ export interface OffsetPaginationConfig {
*/
export interface CursorPaginationConfig {
strategy: PaginationStrategy.CURSOR;
cursorParam?: string; // Parameter name for cursor (default: "cursor")
cursorPath?: string; // JSON path to next cursor (default: "nextCursor")
hasMorePath?: string; // JSON path to check if more items exist (default: "hasMore")
cursorParam?: string; // Parameter name for cursor (default: "cursor")
cursorPath?: string; // JSON path to next cursor (default: "nextCursor")
hasMorePath?: string; // JSON path to check if more items exist (default: "hasMore")
}
/**
@@ -47,21 +47,28 @@ export interface LinkPaginationConfig {
export interface CustomPaginationConfig {
strategy: PaginationStrategy.CUSTOM;
hasNextPage: (response: ICoreResponse<any>) => boolean;
getNextPageParams: (response: ICoreResponse<any>, currentParams: Record<string, string>) => Record<string, string>;
getNextPageParams: (
response: ICoreResponse<any>,
currentParams: Record<string, string>,
) => Record<string, string>;
}
/**
* Union type of all pagination configurations
*/
export type TPaginationConfig = OffsetPaginationConfig | CursorPaginationConfig | LinkPaginationConfig | CustomPaginationConfig;
export type TPaginationConfig =
| OffsetPaginationConfig
| CursorPaginationConfig
| LinkPaginationConfig
| CustomPaginationConfig;
/**
* Interface for a paginated response
*/
export interface TPaginatedResponse<T> {
items: T[]; // Current page items
hasNextPage: boolean; // Whether there are more pages
getNextPage: () => Promise<TPaginatedResponse<T>>; // Function to get the next page
getAllPages: () => Promise<T[]>; // Function to get all remaining pages and combine
response: ICoreResponse<any>; // Original response
}
items: T[]; // Current page items
hasNextPage: boolean; // Whether there are more pages
getNextPage: () => Promise<TPaginatedResponse<T>>; // Function to get the next page
getAllPages: () => Promise<T[]>; // Function to get all remaining pages and combine
response: ICoreResponse<any>; // Original response
}

4
ts/core/index.ts
View File

@@ -13,8 +13,8 @@ if (smartenvInstance.isNode) {
// In Node.js, load the node implementation
const modulePath = plugins.smartpath.join(
plugins.smartpath.dirname(import.meta.url),
'../core_node/index.js'
)
'../core_node/index.js',
);
console.log(modulePath);
const impl = await smartenvInstance.getSafeNodeModule(modulePath);
CoreRequest = impl.CoreRequest;

2
ts/core_base/index.ts
View File

@@ -1,4 +1,4 @@
// Core base exports - abstract classes and platform-agnostic types
export * from './types.js';
export * from './request.js';
export * from './response.js';
export * from './response.js';

8
ts/core_base/request.ts
View File

@@ -3,7 +3,10 @@ import * as types from './types.js';
/**
* Abstract Core Request class that defines the interface for all HTTP/HTTPS requests
*/
export abstract class CoreRequest<TOptions extends types.ICoreRequestOptions = types.ICoreRequestOptions, TResponse = any> {
export abstract class CoreRequest<
TOptions extends types.ICoreRequestOptions = types.ICoreRequestOptions,
TResponse = any,
> {
/**
* Tests if a URL is a unix socket
*/
@@ -41,5 +44,4 @@ export abstract class CoreRequest<TOptions extends types.ICoreRequestOptions = t
* Fire the request and return the raw response (platform-specific)
*/
abstract fireCore(): Promise<any>;
}
}

4
ts/core_base/response.ts
View File

@@ -37,9 +37,9 @@ export abstract class CoreResponse<T = any> implements types.ICoreResponse<T> {
* Get response as ArrayBuffer
*/
abstract arrayBuffer(): Promise<ArrayBuffer>;
/**
* Get response as a web-style ReadableStream
*/
abstract stream(): ReadableStream<Uint8Array> | null;
}
}

17
ts/core_base/types.ts
View File

@@ -1,7 +1,14 @@
/**
* HTTP Methods supported
*/
export type THttpMethod = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS';
export type THttpMethod =
| 'GET'
| 'POST'
| 'PUT'
| 'DELETE'
| 'PATCH'
| 'HEAD'
| 'OPTIONS';
/**
* Response types supported
@@ -39,14 +46,14 @@ export interface ICoreRequestOptions {
timeout?: number;
hardDataCuttingTimeout?: number;
autoDrain?: boolean; // Auto-drain unconsumed responses (Node.js only, default: true)
// Node.js specific options (ignored in fetch implementation)
agent?: any;
socketPath?: string;
hostname?: string;
port?: number;
path?: string;
// Fetch API specific options (ignored in Node.js implementation)
credentials?: RequestCredentials;
mode?: RequestMode;
@@ -73,10 +80,10 @@ export interface ICoreResponse<T = any> {
statusText: string;
headers: Headers;
url: string;
// Methods
json(): Promise<T>;
text(): Promise<string>;
arrayBuffer(): Promise<ArrayBuffer>;
stream(): ReadableStream<Uint8Array> | null; // Always returns web-style stream
}
}

2
ts/core_fetch/index.ts
View File

@@ -1,3 +1,3 @@
// Core fetch exports - native fetch implementation
export * from './response.js';
export { CoreRequest } from './request.js';
export { CoreRequest } from './request.js';

42
ts/core_fetch/request.ts
View File

@@ -5,13 +5,18 @@ import { CoreRequest as AbstractCoreRequest } from '../core_base/request.js';
/**
* Fetch-based implementation of Core Request class
*/
export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions, CoreResponse> {
export class CoreRequest extends AbstractCoreRequest<
types.ICoreRequestOptions,
CoreResponse
> {
constructor(url: string, options: types.ICoreRequestOptions = {}) {
super(url, options);
// Check for unsupported Node.js-specific options
if (options.agent || options.socketPath) {
throw new Error('Node.js specific options (agent, socketPath) are not supported in browser/fetch implementation');
throw new Error(
'Node.js specific options (agent, socketPath) are not supported in browser/fetch implementation',
);
}
}
@@ -19,7 +24,10 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
* Build the full URL with query parameters
*/
private buildUrl(): string {
if (!this.options.queryParams || Object.keys(this.options.queryParams).length === 0) {
if (
!this.options.queryParams ||
Object.keys(this.options.queryParams).length === 0
) {
return this.url;
}
@@ -50,11 +58,13 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
// Handle request body
if (this.options.requestBody !== undefined) {
if (typeof this.options.requestBody === 'string' ||
this.options.requestBody instanceof ArrayBuffer ||
this.options.requestBody instanceof FormData ||
this.options.requestBody instanceof URLSearchParams ||
this.options.requestBody instanceof ReadableStream) {
if (
typeof this.options.requestBody === 'string' ||
this.options.requestBody instanceof ArrayBuffer ||
this.options.requestBody instanceof FormData ||
this.options.requestBody instanceof URLSearchParams ||
this.options.requestBody instanceof ReadableStream
) {
fetchOptions.body = this.options.requestBody;
} else {
// Convert objects to JSON
@@ -66,7 +76,10 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
if (!fetchOptions.headers.has('Content-Type')) {
fetchOptions.headers.set('Content-Type', 'application/json');
}
} else if (typeof fetchOptions.headers === 'object' && !Array.isArray(fetchOptions.headers)) {
} else if (
typeof fetchOptions.headers === 'object' &&
!Array.isArray(fetchOptions.headers)
) {
const headersObj = fetchOptions.headers as Record<string, string>;
if (!headersObj['Content-Type']) {
headersObj['Content-Type'] = 'application/json';
@@ -77,7 +90,8 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
// Handle timeout
if (this.options.timeout || this.options.hardDataCuttingTimeout) {
const timeout = this.options.hardDataCuttingTimeout || this.options.timeout;
const timeout =
this.options.hardDataCuttingTimeout || this.options.timeout;
const controller = new AbortController();
setTimeout(() => controller.abort(), timeout);
fetchOptions.signal = controller.signal;
@@ -100,7 +114,7 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
async fireCore(): Promise<Response> {
const url = this.buildUrl();
const options = this.buildFetchOptions();
try {
const response = await fetch(url, options);
return response;
@@ -117,7 +131,7 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
*/
static async create(
url: string,
options: types.ICoreRequestOptions = {}
options: types.ICoreRequestOptions = {},
): Promise<CoreResponse> {
const request = new CoreRequest(url, options);
return request.fire();
@@ -128,4 +142,4 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
* Convenience exports for backward compatibility
*/
export const isUnixSocket = CoreRequest.isUnixSocket;
export const parseUnixSocketUrl = CoreRequest.parseUnixSocketUrl;
export const parseUnixSocketUrl = CoreRequest.parseUnixSocketUrl;

17
ts/core_fetch/response.ts
View File

@@ -4,7 +4,10 @@ import { CoreResponse as AbstractCoreResponse } from '../core_base/response.js';
/**
* Fetch-based implementation of Core Response class
*/
export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements types.IFetchResponse<T> {
export class CoreResponse<T = any>
extends AbstractCoreResponse<T>
implements types.IFetchResponse<T>
{
private response: Response;
private responseClone: Response;
@@ -20,12 +23,12 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
// Clone the response so we can read the body multiple times if needed
this.response = response;
this.responseClone = response.clone();
this.ok = response.ok;
this.status = response.status;
this.statusText = response.statusText;
this.url = response.url;
// Convert Headers to plain object
this.headers = {};
response.headers.forEach((value, key) => {
@@ -73,13 +76,15 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
* Node.js stream method - not available in browser
*/
streamNode(): never {
throw new Error('streamNode() is not available in browser/fetch environment. Use stream() for web-style ReadableStream.');
throw new Error(
'streamNode() is not available in browser/fetch environment. Use stream() for web-style ReadableStream.',
);
}
/**
* Get the raw Response object
*/
raw(): Response {
return this.responseClone;
}
}
}

4
ts/core_fetch/types.ts
View File

@@ -9,7 +9,7 @@ export * from '../core_base/types.js';
export interface IFetchResponse<T = any> extends baseTypes.ICoreResponse<T> {
// Node.js stream method that throws in browser
streamNode(): never;
// Access to raw Response object
raw(): Response;
}
}

2
ts/core_node/index.ts
View File

@@ -1,3 +1,3 @@
// Core exports
export * from './response.js';
export { CoreRequest } from './request.js';
export { CoreRequest } from './request.js';

2
ts/core_node/plugins.ts
View File

@@ -17,4 +17,4 @@ import { HttpAgent, HttpsAgent } from 'agentkeepalive';
const agentkeepalive = { HttpAgent, HttpsAgent };
import formData from 'form-data';
export { agentkeepalive, formData };
export { agentkeepalive, formData };

56
ts/core_node/request.ts
View File

@@ -29,21 +29,33 @@ const httpsAgentKeepAliveFalse = new plugins.agentkeepalive.HttpsAgent({
/**
* Node.js implementation of Core Request class that handles all HTTP/HTTPS requests
*/
export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions, CoreResponse> {
export class CoreRequest extends AbstractCoreRequest<
types.ICoreRequestOptions,
CoreResponse
> {
private requestDataFunc: ((req: plugins.http.ClientRequest) => void) | null;
constructor(
url: string,
options: types.ICoreRequestOptions = {},
requestDataFunc: ((req: plugins.http.ClientRequest) => void) | null = null
requestDataFunc: ((req: plugins.http.ClientRequest) => void) | null = null,
) {
super(url, options);
this.requestDataFunc = requestDataFunc;
// Check for unsupported fetch-specific options
if (options.credentials || options.mode || options.cache || options.redirect ||
options.referrer || options.referrerPolicy || options.integrity) {
throw new Error('Fetch API specific options (credentials, mode, cache, redirect, referrer, referrerPolicy, integrity) are not supported in Node.js implementation');
if (
options.credentials ||
options.mode ||
options.cache ||
options.redirect ||
options.referrer ||
options.referrerPolicy ||
options.integrity
) {
throw new Error(
'Fetch API specific options (credentials, mode, cache, redirect, referrer, referrerPolicy, integrity) are not supported in Node.js implementation',
);
}
}
@@ -65,7 +77,7 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
const parsedUrl = plugins.smarturl.Smarturl.createFromUrl(this.url, {
searchParams: this.options.queryParams || {},
});
this.options.hostname = parsedUrl.hostname;
if (parsedUrl.port) {
this.options.port = parseInt(parsedUrl.port, 10);
@@ -74,7 +86,9 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
// Handle unix socket URLs
if (CoreRequest.isUnixSocket(this.url)) {
const { socketPath, path } = CoreRequest.parseUnixSocketUrl(this.options.path);
const { socketPath, path } = CoreRequest.parseUnixSocketUrl(
this.options.path,
);
this.options.socketPath = socketPath;
this.options.path = path;
}
@@ -83,18 +97,25 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
if (!this.options.agent) {
// Only use keep-alive agents if explicitly requested
if (this.options.keepAlive === true) {
this.options.agent = parsedUrl.protocol === 'https:' ? httpsAgent : httpAgent;
this.options.agent =
parsedUrl.protocol === 'https:' ? httpsAgent : httpAgent;
} else if (this.options.keepAlive === false) {
this.options.agent = parsedUrl.protocol === 'https:' ? httpsAgentKeepAliveFalse : httpAgentKeepAliveFalse;
this.options.agent =
parsedUrl.protocol === 'https:'
? httpsAgentKeepAliveFalse
: httpAgentKeepAliveFalse;
}
// If keepAlive is undefined, don't set any agent (more fetch-like behavior)
}
// Determine request module
const requestModule = parsedUrl.protocol === 'https:' ? plugins.https : plugins.http;
const requestModule =
parsedUrl.protocol === 'https:' ? plugins.https : plugins.http;
if (!requestModule) {
throw new Error(`The request to ${this.url} is missing a viable protocol. Must be http or https`);
throw new Error(
`The request to ${this.url} is missing a viable protocol. Must be http or https`,
);
}
// Perform the request
@@ -119,11 +140,12 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
});
} else {
// Write body as-is - caller is responsible for serialization
const bodyData = typeof this.options.requestBody === 'string'
? this.options.requestBody
: this.options.requestBody instanceof Buffer
const bodyData =
typeof this.options.requestBody === 'string'
? this.options.requestBody
: JSON.stringify(this.options.requestBody); // Still stringify for backward compatibility
: this.options.requestBody instanceof Buffer
? this.options.requestBody
: JSON.stringify(this.options.requestBody); // Still stringify for backward compatibility
request.write(bodyData);
request.end();
}
@@ -155,7 +177,7 @@ export class CoreRequest extends AbstractCoreRequest<types.ICoreRequestOptions,
*/
static async create(
url: string,
options: types.ICoreRequestOptions = {}
options: types.ICoreRequestOptions = {},
): Promise<CoreResponse> {
const request = new CoreRequest(url, options);
return request.fire();

51
ts/core_node/response.ts
View File

@@ -5,7 +5,10 @@ import { CoreResponse as AbstractCoreResponse } from '../core_base/response.js';
/**
* Node.js implementation of Core Response class that provides a fetch-like API
*/
export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements types.INodeResponse<T> {
export class CoreResponse<T = any>
extends AbstractCoreResponse<T>
implements types.INodeResponse<T>
{
private incomingMessage: plugins.http.IncomingMessage;
private bodyBufferPromise: Promise<Buffer> | null = null;
private _autoDrainTimeout: NodeJS.Immediate | null = null;
@@ -17,7 +20,11 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
public readonly headers: plugins.http.IncomingHttpHeaders;
public readonly url: string;
constructor(incomingMessage: plugins.http.IncomingMessage, url: string, options: types.ICoreRequestOptions = {}) {
constructor(
incomingMessage: plugins.http.IncomingMessage,
url: string,
options: types.ICoreRequestOptions = {},
) {
super();
this.incomingMessage = incomingMessage;
this.url = url;
@@ -25,14 +32,16 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
this.statusText = incomingMessage.statusMessage || '';
this.ok = this.status >= 200 && this.status < 300;
this.headers = incomingMessage.headers;
// Auto-drain unconsumed streams to prevent socket hanging
// This prevents keep-alive sockets from timing out when response bodies aren't consumed
// Default to true if not specified
if (options.autoDrain !== false) {
this._autoDrainTimeout = setImmediate(() => {
if (!this.consumed && !this.incomingMessage.readableEnded) {
console.log(`Auto-draining unconsumed response body for ${this.url} (status: ${this.status})`);
console.log(
`Auto-draining unconsumed response body for ${this.url} (status: ${this.status})`,
);
this.incomingMessage.resume(); // Drain without processing
}
});
@@ -48,7 +57,7 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
clearImmediate(this._autoDrainTimeout);
this._autoDrainTimeout = null;
}
super.ensureNotConsumed();
}
@@ -57,22 +66,22 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
*/
private async collectBody(): Promise<Buffer> {
this.ensureNotConsumed();
if (this.bodyBufferPromise) {
return this.bodyBufferPromise;
}
this.bodyBufferPromise = new Promise<Buffer>((resolve, reject) => {
const chunks: Buffer[] = [];
this.incomingMessage.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
this.incomingMessage.on('end', () => {
resolve(Buffer.concat(chunks));
});
this.incomingMessage.on('error', reject);
});
@@ -85,7 +94,7 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
async json(): Promise<T> {
const buffer = await this.collectBody();
const text = buffer.toString('utf-8');
try {
return JSON.parse(text);
} catch (error) {
@@ -106,7 +115,10 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
*/
async arrayBuffer(): Promise<ArrayBuffer> {
const buffer = await this.collectBody();
return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength,
);
}
/**
@@ -114,13 +126,13 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
*/
stream(): ReadableStream<Uint8Array> | null {
this.ensureNotConsumed();
// Convert Node.js stream to web stream
// In Node.js 16.5+ we can use Readable.toWeb()
if (this.incomingMessage.readableEnded || this.incomingMessage.destroyed) {
return null;
}
// Create a web ReadableStream from the Node.js stream
const nodeStream = this.incomingMessage;
return new ReadableStream<Uint8Array>({
@@ -128,22 +140,22 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
nodeStream.on('data', (chunk) => {
controller.enqueue(new Uint8Array(chunk));
});
nodeStream.on('end', () => {
controller.close();
});
nodeStream.on('error', (err) => {
controller.error(err);
});
},
cancel() {
nodeStream.destroy();
}
},
});
}
/**
* Get response as a Node.js readable stream
*/
@@ -158,5 +170,4 @@ export class CoreResponse<T = any> extends AbstractCoreResponse<T> implements ty
raw(): plugins.http.IncomingMessage {
return this.incomingMessage;
}
}
}

7
ts/core_node/types.ts
View File

@@ -7,7 +7,8 @@ export * from '../core_base/types.js';
/**
* Extended IncomingMessage with body property (legacy compatibility)
*/
export interface IExtendedIncomingMessage<T = any> extends plugins.http.IncomingMessage {
export interface IExtendedIncomingMessage<T = any>
extends plugins.http.IncomingMessage {
body: T;
}
@@ -17,7 +18,7 @@ export interface IExtendedIncomingMessage<T = any> extends plugins.http.Incoming
export interface INodeResponse<T = any> extends baseTypes.ICoreResponse<T> {
// Node.js specific methods
streamNode(): NodeJS.ReadableStream; // Returns Node.js style stream
// Legacy compatibility
raw(): plugins.http.IncomingMessage;
}
}

2
ts/index.ts
View File

@@ -7,4 +7,4 @@ export type { ICoreRequestOptions, ICoreResponse } from './core_base/types.js';
// Default export for easier importing
import { SmartRequest } from './client/smartrequest.js';
export default SmartRequest;
export default SmartRequest;