feat(performance): Add async utility functions and filesystem utilities
- Implemented async utilities including delay, retryWithBackoff, withTimeout, parallelLimit, debounceAsync, AsyncMutex, and CircuitBreaker. - Created tests for async utilities to ensure functionality and reliability. - Developed AsyncFileSystem class with methods for file and directory operations, including ensureDir, readFile, writeFile, remove, and more. - Added tests for filesystem utilities to validate file operations and error handling.
This commit is contained in:
275
ts/core/utils/async-utils.ts
Normal file
275
ts/core/utils/async-utils.ts
Normal file
@ -0,0 +1,275 @@
|
||||
/**
|
||||
* Async utility functions for SmartProxy
|
||||
* Provides non-blocking alternatives to synchronous operations
|
||||
*/
|
||||
|
||||
/**
|
||||
* Delays execution for the specified number of milliseconds
|
||||
* Non-blocking alternative to busy wait loops
|
||||
* @param ms - Number of milliseconds to delay
|
||||
* @returns Promise that resolves after the delay
|
||||
*/
|
||||
export async function delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry an async operation with exponential backoff
|
||||
* @param fn - The async function to retry
|
||||
* @param options - Retry options
|
||||
* @returns The result of the function or throws the last error
|
||||
*/
|
||||
export async function retryWithBackoff<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: {
|
||||
maxAttempts?: number;
|
||||
initialDelay?: number;
|
||||
maxDelay?: number;
|
||||
factor?: number;
|
||||
onRetry?: (attempt: number, error: Error) => void;
|
||||
} = {}
|
||||
): Promise<T> {
|
||||
const {
|
||||
maxAttempts = 3,
|
||||
initialDelay = 100,
|
||||
maxDelay = 10000,
|
||||
factor = 2,
|
||||
onRetry
|
||||
} = options;
|
||||
|
||||
let lastError: Error | null = null;
|
||||
let currentDelay = initialDelay;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (error: any) {
|
||||
lastError = error;
|
||||
|
||||
if (attempt === maxAttempts) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (onRetry) {
|
||||
onRetry(attempt, error);
|
||||
}
|
||||
|
||||
await delay(currentDelay);
|
||||
currentDelay = Math.min(currentDelay * factor, maxDelay);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Retry failed');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute an async operation with a timeout
|
||||
* @param fn - The async function to execute
|
||||
* @param timeoutMs - Timeout in milliseconds
|
||||
* @param timeoutError - Optional custom timeout error
|
||||
* @returns The result of the function or throws timeout error
|
||||
*/
|
||||
export async function withTimeout<T>(
|
||||
fn: () => Promise<T>,
|
||||
timeoutMs: number,
|
||||
timeoutError?: Error
|
||||
): Promise<T> {
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(timeoutError || new Error(`Operation timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
});
|
||||
|
||||
return Promise.race([fn(), timeoutPromise]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run multiple async operations in parallel with a concurrency limit
|
||||
* @param items - Array of items to process
|
||||
* @param fn - Async function to run for each item
|
||||
* @param concurrency - Maximum number of concurrent operations
|
||||
* @returns Array of results in the same order as input
|
||||
*/
|
||||
export async function parallelLimit<T, R>(
|
||||
items: T[],
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
concurrency: number
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
const executing: Set<Promise<void>> = new Set();
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const promise = fn(items[i], i).then(result => {
|
||||
results[i] = result;
|
||||
executing.delete(promise);
|
||||
});
|
||||
|
||||
executing.add(promise);
|
||||
|
||||
if (executing.size >= concurrency) {
|
||||
await Promise.race(executing);
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(executing);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Debounce an async function
|
||||
* @param fn - The async function to debounce
|
||||
* @param delayMs - Delay in milliseconds
|
||||
* @returns Debounced function with cancel method
|
||||
*/
|
||||
export function debounceAsync<T extends (...args: any[]) => Promise<any>>(
|
||||
fn: T,
|
||||
delayMs: number
|
||||
): T & { cancel: () => void } {
|
||||
let timeoutId: NodeJS.Timeout | null = null;
|
||||
let lastPromise: Promise<any> | null = null;
|
||||
|
||||
const debounced = ((...args: Parameters<T>) => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
|
||||
lastPromise = new Promise((resolve, reject) => {
|
||||
timeoutId = setTimeout(async () => {
|
||||
timeoutId = null;
|
||||
try {
|
||||
const result = await fn(...args);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
}, delayMs);
|
||||
});
|
||||
|
||||
return lastPromise;
|
||||
}) as any;
|
||||
|
||||
debounced.cancel = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
timeoutId = null;
|
||||
}
|
||||
};
|
||||
|
||||
return debounced as T & { cancel: () => void };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a mutex for ensuring exclusive access to a resource
|
||||
*/
|
||||
export class AsyncMutex {
|
||||
private queue: Array<() => void> = [];
|
||||
private locked = false;
|
||||
|
||||
async acquire(): Promise<() => void> {
|
||||
if (!this.locked) {
|
||||
this.locked = true;
|
||||
return () => this.release();
|
||||
}
|
||||
|
||||
return new Promise<() => void>(resolve => {
|
||||
this.queue.push(() => {
|
||||
resolve(() => this.release());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private release(): void {
|
||||
const next = this.queue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
this.locked = false;
|
||||
}
|
||||
}
|
||||
|
||||
async runExclusive<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const release = await this.acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Circuit breaker for protecting against cascading failures
|
||||
*/
|
||||
export class CircuitBreaker {
|
||||
private failureCount = 0;
|
||||
private lastFailureTime = 0;
|
||||
private state: 'closed' | 'open' | 'half-open' = 'closed';
|
||||
|
||||
constructor(
|
||||
private options: {
|
||||
failureThreshold: number;
|
||||
resetTimeout: number;
|
||||
onStateChange?: (state: 'closed' | 'open' | 'half-open') => void;
|
||||
}
|
||||
) {}
|
||||
|
||||
async execute<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (this.state === 'open') {
|
||||
if (Date.now() - this.lastFailureTime > this.options.resetTimeout) {
|
||||
this.setState('half-open');
|
||||
} else {
|
||||
throw new Error('Circuit breaker is open');
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await fn();
|
||||
this.onSuccess();
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.onFailure();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private onSuccess(): void {
|
||||
this.failureCount = 0;
|
||||
if (this.state !== 'closed') {
|
||||
this.setState('closed');
|
||||
}
|
||||
}
|
||||
|
||||
private onFailure(): void {
|
||||
this.failureCount++;
|
||||
this.lastFailureTime = Date.now();
|
||||
|
||||
if (this.failureCount >= this.options.failureThreshold) {
|
||||
this.setState('open');
|
||||
}
|
||||
}
|
||||
|
||||
private setState(state: 'closed' | 'open' | 'half-open'): void {
|
||||
if (this.state !== state) {
|
||||
this.state = state;
|
||||
if (this.options.onStateChange) {
|
||||
this.options.onStateChange(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.state === 'open';
|
||||
}
|
||||
|
||||
getState(): 'closed' | 'open' | 'half-open' {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
recordSuccess(): void {
|
||||
this.onSuccess();
|
||||
}
|
||||
|
||||
recordFailure(): void {
|
||||
this.onFailure();
|
||||
}
|
||||
}
|
||||
270
ts/core/utils/fs-utils.ts
Normal file
270
ts/core/utils/fs-utils.ts
Normal file
@ -0,0 +1,270 @@
|
||||
/**
|
||||
* Async filesystem utilities for SmartProxy
|
||||
* Provides non-blocking alternatives to synchronous filesystem operations
|
||||
*/
|
||||
|
||||
import * as plugins from '../../plugins.js';
|
||||
|
||||
export class AsyncFileSystem {
|
||||
/**
|
||||
* Check if a file or directory exists
|
||||
* @param path - Path to check
|
||||
* @returns Promise resolving to true if exists, false otherwise
|
||||
*/
|
||||
static async exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await plugins.fs.promises.access(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a directory exists, creating it if necessary
|
||||
* @param dirPath - Directory path to ensure
|
||||
* @returns Promise that resolves when directory is ensured
|
||||
*/
|
||||
static async ensureDir(dirPath: string): Promise<void> {
|
||||
await plugins.fs.promises.mkdir(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a file as string
|
||||
* @param filePath - Path to the file
|
||||
* @param encoding - File encoding (default: utf8)
|
||||
* @returns Promise resolving to file contents
|
||||
*/
|
||||
static async readFile(filePath: string, encoding: BufferEncoding = 'utf8'): Promise<string> {
|
||||
return plugins.fs.promises.readFile(filePath, encoding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a file as buffer
|
||||
* @param filePath - Path to the file
|
||||
* @returns Promise resolving to file buffer
|
||||
*/
|
||||
static async readFileBuffer(filePath: string): Promise<Buffer> {
|
||||
return plugins.fs.promises.readFile(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write string data to a file
|
||||
* @param filePath - Path to the file
|
||||
* @param data - String data to write
|
||||
* @param encoding - File encoding (default: utf8)
|
||||
* @returns Promise that resolves when file is written
|
||||
*/
|
||||
static async writeFile(filePath: string, data: string, encoding: BufferEncoding = 'utf8'): Promise<void> {
|
||||
// Ensure directory exists
|
||||
const dir = plugins.path.dirname(filePath);
|
||||
await this.ensureDir(dir);
|
||||
await plugins.fs.promises.writeFile(filePath, data, encoding);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write buffer data to a file
|
||||
* @param filePath - Path to the file
|
||||
* @param data - Buffer data to write
|
||||
* @returns Promise that resolves when file is written
|
||||
*/
|
||||
static async writeFileBuffer(filePath: string, data: Buffer): Promise<void> {
|
||||
const dir = plugins.path.dirname(filePath);
|
||||
await this.ensureDir(dir);
|
||||
await plugins.fs.promises.writeFile(filePath, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a file
|
||||
* @param filePath - Path to the file
|
||||
* @returns Promise that resolves when file is removed
|
||||
*/
|
||||
static async remove(filePath: string): Promise<void> {
|
||||
try {
|
||||
await plugins.fs.promises.unlink(filePath);
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
throw error;
|
||||
}
|
||||
// File doesn't exist, which is fine
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a directory and all its contents
|
||||
* @param dirPath - Path to the directory
|
||||
* @returns Promise that resolves when directory is removed
|
||||
*/
|
||||
static async removeDir(dirPath: string): Promise<void> {
|
||||
try {
|
||||
await plugins.fs.promises.rm(dirPath, { recursive: true, force: true });
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read JSON from a file
|
||||
* @param filePath - Path to the JSON file
|
||||
* @returns Promise resolving to parsed JSON
|
||||
*/
|
||||
static async readJSON<T = any>(filePath: string): Promise<T> {
|
||||
const content = await this.readFile(filePath);
|
||||
return JSON.parse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write JSON to a file
|
||||
* @param filePath - Path to the file
|
||||
* @param data - Data to write as JSON
|
||||
* @param pretty - Whether to pretty-print JSON (default: true)
|
||||
* @returns Promise that resolves when file is written
|
||||
*/
|
||||
static async writeJSON(filePath: string, data: any, pretty = true): Promise<void> {
|
||||
const jsonString = pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
|
||||
await this.writeFile(filePath, jsonString);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a file from source to destination
|
||||
* @param source - Source file path
|
||||
* @param destination - Destination file path
|
||||
* @returns Promise that resolves when file is copied
|
||||
*/
|
||||
static async copyFile(source: string, destination: string): Promise<void> {
|
||||
const destDir = plugins.path.dirname(destination);
|
||||
await this.ensureDir(destDir);
|
||||
await plugins.fs.promises.copyFile(source, destination);
|
||||
}
|
||||
|
||||
/**
|
||||
* Move/rename a file
|
||||
* @param source - Source file path
|
||||
* @param destination - Destination file path
|
||||
* @returns Promise that resolves when file is moved
|
||||
*/
|
||||
static async moveFile(source: string, destination: string): Promise<void> {
|
||||
const destDir = plugins.path.dirname(destination);
|
||||
await this.ensureDir(destDir);
|
||||
await plugins.fs.promises.rename(source, destination);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file stats
|
||||
* @param filePath - Path to the file
|
||||
* @returns Promise resolving to file stats or null if doesn't exist
|
||||
*/
|
||||
static async getStats(filePath: string): Promise<plugins.fs.Stats | null> {
|
||||
try {
|
||||
return await plugins.fs.promises.stat(filePath);
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in a directory
|
||||
* @param dirPath - Directory path
|
||||
* @returns Promise resolving to array of filenames
|
||||
*/
|
||||
static async listFiles(dirPath: string): Promise<string[]> {
|
||||
try {
|
||||
return await plugins.fs.promises.readdir(dirPath);
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in a directory with full paths
|
||||
* @param dirPath - Directory path
|
||||
* @returns Promise resolving to array of full file paths
|
||||
*/
|
||||
static async listFilesFullPath(dirPath: string): Promise<string[]> {
|
||||
const files = await this.listFiles(dirPath);
|
||||
return files.map(file => plugins.path.join(dirPath, file));
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively list all files in a directory
|
||||
* @param dirPath - Directory path
|
||||
* @param fileList - Accumulator for file list (used internally)
|
||||
* @returns Promise resolving to array of all file paths
|
||||
*/
|
||||
static async listFilesRecursive(dirPath: string, fileList: string[] = []): Promise<string[]> {
|
||||
const files = await this.listFiles(dirPath);
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = plugins.path.join(dirPath, file);
|
||||
const stats = await this.getStats(filePath);
|
||||
|
||||
if (stats?.isDirectory()) {
|
||||
await this.listFilesRecursive(filePath, fileList);
|
||||
} else if (stats?.isFile()) {
|
||||
fileList.push(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
return fileList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a read stream for a file
|
||||
* @param filePath - Path to the file
|
||||
* @param options - Stream options
|
||||
* @returns Read stream
|
||||
*/
|
||||
static createReadStream(filePath: string, options?: Parameters<typeof plugins.fs.createReadStream>[1]): plugins.fs.ReadStream {
|
||||
return plugins.fs.createReadStream(filePath, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a write stream for a file
|
||||
* @param filePath - Path to the file
|
||||
* @param options - Stream options
|
||||
* @returns Write stream
|
||||
*/
|
||||
static createWriteStream(filePath: string, options?: Parameters<typeof plugins.fs.createWriteStream>[1]): plugins.fs.WriteStream {
|
||||
return plugins.fs.createWriteStream(filePath, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a file exists, creating an empty file if necessary
|
||||
* @param filePath - Path to the file
|
||||
* @returns Promise that resolves when file is ensured
|
||||
*/
|
||||
static async ensureFile(filePath: string): Promise<void> {
|
||||
const exists = await this.exists(filePath);
|
||||
if (!exists) {
|
||||
await this.writeFile(filePath, '');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is a directory
|
||||
* @param path - Path to check
|
||||
* @returns Promise resolving to true if directory, false otherwise
|
||||
*/
|
||||
static async isDirectory(path: string): Promise<boolean> {
|
||||
const stats = await this.getStats(path);
|
||||
return stats?.isDirectory() ?? false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is a file
|
||||
* @param path - Path to check
|
||||
* @returns Promise resolving to true if file, false otherwise
|
||||
*/
|
||||
static async isFile(path: string): Promise<boolean> {
|
||||
const stats = await this.getStats(path);
|
||||
return stats?.isFile() ?? false;
|
||||
}
|
||||
}
|
||||
@ -13,3 +13,5 @@ export * from './shared-security-manager.js';
|
||||
export * from './event-system.js';
|
||||
export * from './websocket-utils.js';
|
||||
export * from './logger.js';
|
||||
export * from './async-utils.js';
|
||||
export * from './fs-utils.js';
|
||||
|
||||
Reference in New Issue
Block a user