feat(core): Add concurrent processing support

This commit is contained in:
2025-01-04 13:11:58 +01:00
parent e0a9030bcd
commit 7b3965ab2b
6 changed files with 7128 additions and 2758 deletions

View File

@@ -1,8 +1,8 @@
/**
* autocreated commitinfo by @pushrocks/commitinfo
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/smartarray',
version: '1.0.8',
version: '1.1.0',
description: 'A TypeScript library for enhancing array manipulation with asynchronous operations such as mapping, filtering, and deduplication.'
}

View File

@@ -0,0 +1,35 @@
export class ConcurrentProcessor<T> {
private asyncFunction: (item: T) => Promise<void>;
private concurrencyLimit: number;
constructor(asyncFunction: (item: T) => Promise<void>, concurrencyLimit: number) {
this.asyncFunction = asyncFunction;
this.concurrencyLimit = concurrencyLimit;
}
async process(items: T[]): Promise<void> {
const queue: Array<Promise<void>> = [];
let totalProcessed = 0;
for (const item of items) {
const promise = this.asyncFunction(item).then(() => {
totalProcessed++;
if (totalProcessed % 10000 === 0) {
console.log(`${totalProcessed} items processed.`);
}
// Remove the completed promise from the queue
queue.splice(queue.indexOf(promise), 1);
});
queue.push(promise);
// Wait if the number of promises exceeds the concurrency limit
if (queue.length >= this.concurrencyLimit) {
await Promise.race(queue);
}
}
// Wait for the remaining promises to resolve
await Promise.all(queue);
}
}

View File

@@ -1,3 +1,5 @@
export * from './classes.concurrentprocessor.js';
export const map = async <T = any, R = any>(
arrayArg: T[],
mapFunction: (itemArg: T) => Promise<R>