diff --git a/package.json b/package.json index b16b303..79a5597 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,13 @@ "@git.zone/tsbuild": "^2.1.66", "@git.zone/tsrun": "^1.2.44", "@git.zone/tstest": "^1.0.84", + "@push.rocks/smartdelay": "^3.0.5", "@push.rocks/smartfile": "^11.0.0", "@push.rocks/tapbundle": "^5.0.15", "@types/node": "^20.9.0" }, "dependencies": { + "@push.rocks/lik": "^6.0.6", "@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartrx": "^3.0.7" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e6aeec..dc599a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,9 @@ settings: excludeLinksFromLockfile: false dependencies: + '@push.rocks/lik': + specifier: ^6.0.6 + version: 6.0.6 '@push.rocks/smartpromise': specifier: ^4.0.3 version: 4.0.3 @@ -22,6 +25,9 @@ devDependencies: '@git.zone/tstest': specifier: ^1.0.84 version: 1.0.84(@types/node@20.9.0)(sinon@17.0.1) + '@push.rocks/smartdelay': + specifier: ^3.0.5 + version: 3.0.5 '@push.rocks/smartfile': specifier: ^11.0.0 version: 11.0.0 @@ -573,19 +579,6 @@ packages: resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==} dev: true - /@push.rocks/lik@6.0.5: - resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==} - dependencies: - '@push.rocks/smartdelay': 3.0.5 - '@push.rocks/smartmatch': 2.0.0 - '@push.rocks/smartpromise': 4.0.3 - '@push.rocks/smartrx': 3.0.7 - '@push.rocks/smarttime': 4.0.6 - '@types/minimatch': 5.1.2 - '@types/symbol-tree': 3.2.4 - symbol-tree: 3.2.4 - dev: true - /@push.rocks/lik@6.0.6: resolution: {integrity: sha512-5NVFbtR2XsGHxNqUbjFPp91J7mCWFPUc8bVfXIY2YgzhK/I5XI7yT+QBM+Bc2IvGLxjvQPkueRfPKl1IEnegeA==} dependencies: @@ -597,7 +590,6 @@ packages: '@types/minimatch': 5.1.2 '@types/symbol-tree': 3.2.5 symbol-tree: 3.2.4 - dev: true /@push.rocks/smartbrowser@2.0.6: resolution: {integrity: sha512-Ne+KCVhV/DROc1rHRRw59K6h0+LpQAK9fdOUtgDZ7laLPmB/tmnbUh3IuRDNcIY1iVA9pydoobwjnTjVgio9eQ==} @@ -636,7 +628,7 @@ packages: /@push.rocks/smartcli@4.0.8: resolution: {integrity: sha512-B4F3nqq7ko8tev1wxGdFnh/zSDDP8Q9LpEOb3wTf0jayyhYetFQ7n6zi4J9fhXYBKPkJSyQEBoOfRmgJyeLHkA==} dependencies: - '@push.rocks/lik': 6.0.5 + '@push.rocks/lik': 6.0.6 '@push.rocks/smartlog': 3.0.3 '@push.rocks/smartparam': 1.1.10 '@push.rocks/smartpromise': 4.0.3 @@ -648,7 +640,6 @@ packages: resolution: {integrity: sha512-mUuI7kj2f7ztjpic96FvRIlf2RsKBa5arw81AHNsndbxO6asRcxuWL8dTVxouEIK8YsBUlj0AsrCkHhMbLQdHw==} dependencies: '@push.rocks/smartpromise': 4.0.3 - dev: true /@push.rocks/smartenv@5.0.10: resolution: {integrity: sha512-yJURKoz5awTPyN+5MO1A9maf5+Zh2qutHlbZPx2y5P6rwGPFWThippLicqXUSs2rM49ES6RKAD48Q8w5GQkWFw==} @@ -685,7 +676,7 @@ packages: /@push.rocks/smartfile@10.0.41: resolution: {integrity: sha512-xOOy0duI34M2qrJZggpk51EHGXmg9+mBL1Q55tNiQKXzfx89P3coY1EAZG8tvmep3qB712QEKe7T+u04t42Kjg==} dependencies: - '@push.rocks/lik': 6.0.5 + '@push.rocks/lik': 6.0.6 '@push.rocks/smartdelay': 3.0.5 '@push.rocks/smartfile-interfaces': 1.0.7 '@push.rocks/smarthash': 3.0.4 @@ -778,7 +769,6 @@ packages: resolution: {integrity: sha512-MBzP++1yNIBeox71X6VxpIgZ8m4bXnJpZJ4nWVH6IWpmO38MXTu4X0QF8tQnyT4LFcwvc9iiWaD15cstHa7Mmw==} dependencies: matcher: 5.0.0 - dev: true /@push.rocks/smartmime@1.0.6: resolution: {integrity: sha512-PHd+I4UcsnOATNg8wjDsSAmmJ4CwQFrQCNzd0HSJMs4ZpiK3Ya91almd6GLpDPU370U4HFh4FaPF4eEAI6vkJQ==} @@ -973,14 +963,13 @@ packages: /@push.rocks/smarttime@4.0.6: resolution: {integrity: sha512-1whOow0YJw/TbN758TedRRxApoZbsvyxCVpoGjXh7DE/fEEgs7RCr4vVF5jYpyXNQuNMLpKJcTsSfyQ6RvH4Aw==} dependencies: - '@push.rocks/lik': 6.0.5 + '@push.rocks/lik': 6.0.6 '@push.rocks/smartdelay': 3.0.5 '@push.rocks/smartpromise': 4.0.3 croner: 7.0.4 dayjs: 1.11.10 is-nan: 1.3.2 pretty-ms: 8.0.0 - dev: true /@push.rocks/smartunique@3.0.6: resolution: {integrity: sha512-/fvh5BBHiTvZcCH4km2Oh3f0spfcWKr7e0Dp6lAb3rJq+5KiO7fQmI1UcQ9clADUaaJOn8sPzseiLt+kJ2tajw==} @@ -1664,7 +1653,6 @@ packages: /@types/minimatch@5.1.2: resolution: {integrity: sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==} - dev: true /@types/mocha@5.2.7: resolution: {integrity: sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==} @@ -1763,13 +1751,8 @@ packages: resolution: {integrity: sha512-GDV68H0mBSN449sa5HEj51E0wfpVQb8xNSMzxf/PrypMFcLTMwJMOM/cgXiv71Mq5drkOQmUGvL1okOZcu6RrQ==} dev: true - /@types/symbol-tree@3.2.4: - resolution: {integrity: sha512-j9qIwMi0fGQLssA0/06i8XfqCq9eKsbTu+c9uPx1T9zWisSaFWEE2CwJccn/idKCllXUeHaMSeH/i4HQ6JKEAg==} - dev: true - /@types/symbol-tree@3.2.5: resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==} - dev: true /@types/through2@2.0.40: resolution: {integrity: sha512-QeEwyeGxvtKjP0I/SR8pHtKMgmqotWTI8V/rNMTjbF3arWZJV/kYtbZXtwQKxUuV/oPSGkUilSspFKkTgCPTpA==} @@ -2263,7 +2246,6 @@ packages: function-bind: 1.1.2 get-intrinsic: 1.2.2 set-function-length: 1.1.1 - dev: true /callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} @@ -2501,7 +2483,6 @@ packages: /croner@7.0.4: resolution: {integrity: sha512-P8Zd88km8oQ0xH8Es0u75GtOnFyCNopuAhlFv5kAnbcTuXd0xNvRTgnxnJEs63FicCOsHTL7rpu4BHzY3cMq4w==} engines: {node: '>=6.0'} - dev: true /cross-fetch@3.1.5: resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==} @@ -2529,7 +2510,6 @@ packages: /dayjs@1.11.10: resolution: {integrity: sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==} - dev: true /debounce@1.2.1: resolution: {integrity: sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==} @@ -2617,7 +2597,6 @@ packages: get-intrinsic: 1.2.2 gopd: 1.0.1 has-property-descriptors: 1.0.1 - dev: true /define-lazy-prop@2.0.0: resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} @@ -2631,7 +2610,6 @@ packages: define-data-property: 1.1.1 has-property-descriptors: 1.0.1 object-keys: 1.1.1 - dev: true /delayed-stream@1.0.0: resolution: {integrity: sha1-3zrhmayt+31ECqrgsp4icrJOxhk=} @@ -2904,7 +2882,6 @@ packages: /escape-string-regexp@5.0.0: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - dev: true /esm@3.2.25: resolution: {integrity: sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==} @@ -3164,7 +3141,6 @@ packages: /function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - dev: true /function.prototype.name@1.1.6: resolution: {integrity: sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==} @@ -3196,7 +3172,6 @@ packages: has-proto: 1.0.1 has-symbols: 1.0.3 hasown: 2.0.0 - dev: true /get-stream@5.2.0: resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} @@ -3282,7 +3257,6 @@ packages: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} dependencies: get-intrinsic: 1.2.2 - dev: true /got@12.6.1: resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} @@ -3328,17 +3302,14 @@ packages: resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} dependencies: get-intrinsic: 1.2.2 - dev: true /has-proto@1.0.1: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} - dev: true /has-symbols@1.0.3: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} engines: {node: '>= 0.4'} - dev: true /has-tostringtag@1.0.0: resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} @@ -3352,7 +3323,6 @@ packages: engines: {node: '>= 0.4'} dependencies: function-bind: 1.1.2 - dev: true /he@1.2.0: resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} @@ -3607,7 +3577,6 @@ packages: dependencies: call-bind: 1.0.5 define-properties: 1.2.1 - dev: true /is-negative-zero@2.0.2: resolution: {integrity: sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==} @@ -4088,7 +4057,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: escape-string-regexp: 5.0.0 - dev: true /math-random@1.0.4: resolution: {integrity: sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==} @@ -4340,7 +4308,6 @@ packages: /object-keys@1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} engines: {node: '>= 0.4'} - dev: true /object.assign@4.1.0: resolution: {integrity: sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==} @@ -4472,7 +4439,6 @@ packages: /parse-ms@3.0.0: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} - dev: true /parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} @@ -4591,7 +4557,6 @@ packages: engines: {node: '>=14.16'} dependencies: parse-ms: 3.0.0 - dev: true /process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} @@ -4939,7 +4904,6 @@ packages: get-intrinsic: 1.2.2 gopd: 1.0.1 has-property-descriptors: 1.0.1 - dev: true /set-function-name@2.0.1: resolution: {integrity: sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==} @@ -5257,7 +5221,6 @@ packages: /symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} - dev: true /systeminformation@5.21.16: resolution: {integrity: sha512-WJZdc5RbmDF1VOJcS+G2oIBeNgmMTyJ8+enHe9stIyBg2XhboUTsSfN9HrFMDV+fgXwJ50kNKgD3TzodsFgB+g==} diff --git a/test/backpressure.ts b/test/backpressure.ts new file mode 100644 index 0000000..0312d50 --- /dev/null +++ b/test/backpressure.ts @@ -0,0 +1,57 @@ +import { SmartDuplex, type ISmartDuplexOptions, StreamWrapper } from '../ts/index.js'; + +import * as smartdelay from '@push.rocks/smartdelay'; + +async function testBackpressure() { + const stream1 = new SmartDuplex({ + objectMode: true, + handleBackpressure: true, + writeFunction: async (chunk, tools) => { + console.log(`processed chunk ${chunk} in stream 1`); + return chunk; // Fast processing + } + }); + const stream2 = new SmartDuplex({ + objectMode: true, + handleBackpressure: true, + writeFunction: async (chunk, tools) => { + await new Promise(resolve => setTimeout(resolve, 100)); // Slow processing + console.log(`processed chunk ${chunk} in stream 2`); + return chunk; + } + }); // This stream processes data more slowly + const stream3 = new SmartDuplex({ + handleBackpressure: false, + writeFunction: async (chunk, tools) => { + console.log(`finished chunk ${chunk} in stream 3`); + } + }); + + stream1.pipe(stream2).pipe(stream3); + + let backpressured = false; + for (let i = 1; i < 100; i++) { + const canContinue = stream1.write(`Chunk ${i}`, 'utf8'); + if (!canContinue) { + backpressured = true; + console.log(`Backpressure at chunk ${i}`); + } + } + + stream1.end(); + + stream1.on('finish', () => { + console.log('Stream 1 finished processing.'); + }); + stream2.on('finish', () => { + console.log('Stream 2 finished processing.'); + }); + stream3.on('finish', () => { + console.log('Stream 3 finished processing.'); + if (!backpressured) { + console.log('No backpressure was observed.'); + } + }); +} + +testBackpressure(); \ No newline at end of file diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 9b2f99a..e25bff0 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartstream', - version: '3.0.18', + version: '3.0.19', description: 'simplifies access to node streams' } diff --git a/ts/smartstream.classes.smartduplex.ts b/ts/smartstream.classes.smartduplex.ts index 858a45d..b7b63d3 100644 --- a/ts/smartstream.classes.smartduplex.ts +++ b/ts/smartstream.classes.smartduplex.ts @@ -129,23 +129,7 @@ export class SmartDuplex extends Duplex { } } - public notBackpressured = true; - public get backpressured(): boolean { - return !this.notBackpressured; - } - public push(chunkArg?: TOutput | null): boolean { - const result = super.push(chunkArg); - if (!result && this.handleBackpressure) { - this.notBackpressured = false; - this.pause(); - // Listen for 'drain' event to resume - this.once('drain', () => { - this.notBackpressured = true; - this.resume(); // Resume the source of data - }); - } - return result; - } + private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap>(); // Ensure the _write method types the chunk as TInput and encodes TOutput public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) { @@ -162,26 +146,39 @@ export class SmartDuplex extends Duplex { }; try { + const writeDeferred = plugins.smartpromise.defer(); + this.asyncWritePromiseObjectmap.add(writeDeferred.promise); const modifiedChunk = await this.writeFunction(chunk, tools); if (modifiedChunk) { - this.push(modifiedChunk) - if (this.backpressured && this.handleBackpressure) { - this.once('drain', () => { - callback(); - }); + const drainDeferred = plugins.smartpromise.defer(); + this.once('drain', () => { + drainDeferred.resolve(); + }); + const canPushMore = this.push(modifiedChunk); + if (!canPushMore) { + await drainDeferred.promise; + console.log('jojojo'); + callback(); + writeDeferred.resolve(); } else { callback(); - } + writeDeferred.resolve(); + } } else { callback(); + writeDeferred.resolve(); } + writeDeferred.resolve(); + writeDeferred.promise.then(() => { + this.asyncWritePromiseObjectmap.remove(writeDeferred.promise); + }); } catch (err) { callback(err); } - return this.notBackpressured; } public async _final(callback: (error?: Error | null) => void) { + await Promise.all(this.asyncWritePromiseObjectmap.getArray()); if (this.finalFunction) { const tools: IStreamTools = { truncate: () => callback(), @@ -194,10 +191,10 @@ export class SmartDuplex extends Duplex { this.push(finalChunk); } } catch (err) { + this.push(null); callback(err); + return; } - } else { - // nothing here } this.push(null); callback(); diff --git a/ts/smartstream.plugins.ts b/ts/smartstream.plugins.ts index a3e17e7..c01c2e7 100644 --- a/ts/smartstream.plugins.ts +++ b/ts/smartstream.plugins.ts @@ -4,8 +4,9 @@ import * as stream from 'stream'; export { stream }; // pushrocks scope +import * as lik from '@push.rocks/lik'; import * as smartpromise from '@push.rocks/smartpromise'; import * as smartrx from '@push.rocks/smartrx'; -export { smartpromise, smartrx }; +export { lik, smartpromise, smartrx };