3.0.33

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "3.0.20",
+  "version": "3.0.33",
   "private": false,
   "description": "simplifies access to node streams",
   "main": "dist_ts/index.js",
@@ -22,16 +22,15 @@
   },
   "homepage": "https://gitlab.com/push.rocks/smartstream#readme",
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.66",
+    "@git.zone/tsbuild": "^2.1.72",
     "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.84",
+    "@git.zone/tstest": "^1.0.86",
-    "@push.rocks/smartdelay": "^3.0.5",
+    "@push.rocks/smartfile": "^11.0.4",
-    "@push.rocks/smartfile": "^11.0.0",
     "@push.rocks/tapbundle": "^5.0.15",
-    "@types/node": "^20.9.0"
+    "@types/node": "^20.11.20"
   },
   "dependencies": {
-    "@push.rocks/lik": "^6.0.12",
+    "@push.rocks/lik": "^6.0.14",
     "@push.rocks/smartpromise": "^4.0.3",
     "@push.rocks/smartrx": "^3.0.7"
   },

test/assets/writabletext.txt

@@ -46,3 +46,5 @@ hi+wow
 hi+wow
 hi+wow
 hi+wow
+hi+wow
+hi+wow

test/backpressure.ts

@@ -1,62 +0,0 @@
-import { SmartDuplex, type ISmartDuplexOptions, StreamWrapper } from '../ts/index.js';
-
-import * as smartdelay from '@push.rocks/smartdelay';
-
-async function testBackpressure() {
-  const stream1 = new SmartDuplex({
-    name: 'stream1',
-    objectMode: true,
-    handleBackpressure: true,
-    writeFunction: async (chunk, tools) => {
-      console.log(`processed chunk ${chunk} in stream 1`);
-      return chunk; // Fast processing
-    }
-  });
-  const stream2 = new SmartDuplex({
-    name: 'stream2',
-    objectMode: true,
-    handleBackpressure: true,
-    writeFunction: async (chunk, tools) => {
-      await new Promise(resolve => setTimeout(resolve, 1)); // Slow processing
-      console.log(`processed chunk ${chunk} in stream 2`);
-      return chunk;
-    }
-  }); // This stream processes data more slowly
-  const stream3 = new SmartDuplex({
-    objectMode: true,
-    name: 'stream3',
-    handleBackpressure: true,
-    writeFunction: async (chunk, tools) => {
-      await new Promise(resolve => setTimeout(resolve, 200)); // Slow processing
-      console.log(`processed chunk ${chunk} in stream 3`);
-    }
-  });
-
-  stream1.pipe(stream2).pipe(stream3);
-
-  let backpressured = false;
-  for (let i = 0; i < 1000; i++) {
-    const canContinue = stream1.write(`Chunk ${i}`, 'utf8');
-    if (!canContinue) {
-      backpressured = true;
-      console.log(`Backpressure at chunk ${i}`);
-    }
-  }
-
-  stream1.end();
-
-  stream1.on('finish', () => {
-    console.log('Stream 1 finished processing.');
-  });
-  stream2.on('finish', () => {
-    console.log('Stream 2 finished processing.');
-  });
-  stream3.on('finish', () => {
-    console.log('Stream 3 finished processing.');
-    if (!backpressured) {
-      console.log('No backpressure was observed.');
-    }
-  });
-}
-
-testBackpressure();

test/test.backpressure.ts

@@ -0,0 +1,68 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import { SmartDuplex, type ISmartDuplexOptions, StreamWrapper } from '../ts/index.js';
+
+tap.test('should run backpressure test', async (toolsArg) => {
+  const done = toolsArg.defer();
+  async function testBackpressure() {
+    const stream1 = new SmartDuplex({
+      name: 'stream1',
+      objectMode: true,
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 10)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 1`);
+        return chunk; // Fast processing
+      },
+    });
+    const stream2 = new SmartDuplex({
+      name: 'stream2',
+      objectMode: true,
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 20)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 2`);
+        await tools.push(chunk);
+        // return chunk, optionally return ;
+      },
+    }); // This stream processes data more slowly
+    const stream3 = new SmartDuplex({
+      objectMode: true,
+      name: 'stream3',
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 200)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 3`);
+      },
+    });
+
+    stream1.pipe(stream2).pipe(stream3);
+
+    let backpressured = false;
+    for (let i = 0; i < 200; i++) {
+      const canContinue = stream1.write(`Chunk ${i}`, 'utf8');
+      if (!canContinue) {
+        backpressured = true;
+        console.log(`Backpressure at chunk ${i}`);
+      }
+    }
+
+    stream1.end();
+
+    stream1.on('finish', () => {
+      console.log('Stream 1 finished processing.');
+    });
+    stream2.on('finish', () => {
+      console.log('Stream 2 finished processing.');
+    });
+    stream3.on('finish', () => {
+      console.log('Stream 3 finished processing.');
+      if (!backpressured) {
+        throw new Error('No backpressure was observed.');
+      } else {
+        done.resolve();
+      }
+    });
+  }
+
+  testBackpressure();
+  await done.promise;
+});
+
+await tap.start();

test/test.smartstream.ts

@@ -5,9 +5,7 @@ import * as fs from 'fs';
 
 tap.test('should create a SmartStream from a Buffer', async () => {
   const bufferData = Buffer.from('This is a test buffer');
-  const smartStream = SmartDuplex.fromBuffer(bufferData, {
+  const smartStream = SmartDuplex.fromBuffer(bufferData, {});
-    handleBackpressure: false,
-  });
   
   let receivedData = Buffer.alloc(0);
   

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.20',
+  version: '3.0.33',
   description: 'simplifies access to node streams'
 }

ts/index.ts

@@ -1,4 +1,3 @@
-export * from './smartstream.classes.passthrough.js';
 export * from './smartstream.classes.smartduplex.js';
 export * from './smartstream.classes.streamwrapper.js';
 export * from './smartstream.classes.streamintake.js';

ts/smartstream.classes.passthrough.ts

@@ -1,21 +0,0 @@
-import * as plugins from './smartstream.plugins.js';
-
-export class PassThrough extends plugins.stream.Duplex {
-  constructor(options?: plugins.stream.DuplexOptions) {
-    super(options);
-  }
-
-  _read(size: number): void {
-    // No-op: Data written will be automatically available for reading.
-  }
-
-  _write(chunk: any, encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
-    if (this.push(chunk, encoding)) {
-      callback();
-    } else {
-      this.once('drain', () => {
-        callback();
-      });
-    }
-  }
-}

ts/smartstream.classes.smartduplex.ts

@@ -3,7 +3,7 @@ import { Duplex, type DuplexOptions } from 'stream';
 
 export interface IStreamTools {
   truncate: () => void;
-  push: (pipeObject: any) => void;
+  push: (pipeObject: any) => Promise<void>;
 }
 
 export interface IStreamWriteFunction<T, rT> {
@@ -15,13 +15,30 @@ export interface IStreamFinalFunction<rT> {
 }
 
 export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
+  /**
+   * wether to print debug logs
+   */
   debug?: boolean;
+  /**
+   * the name of the stream
+   */
   name?: string;
-  handleBackpressure?: boolean;
+  /**
+   * a function that is being called to read more stuff from whereever to be processed by the stream
+   * @returns
+   */
   readFunction?: () => Promise<void>;
+
+  /**
+   * the write function is called for every chunk that is being written to the stream
+   * it can push or return chunks (but does not have to) to be written to the readable side of the stream
+   */
   writeFunction?: IStreamWriteFunction<TInput, TOutput>;
+  
+  /**
+   * a final function that is run at the end of the stream 
+   */
   finalFunction?: IStreamFinalFunction<TOutput>;
-  // Add other custom options if necessary
 }
 
 export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
@@ -36,7 +53,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   }
 
   // INSTANCE
-  private backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>();
+  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
   public options: ISmartDuplexOptions<TInput, TOutput>;
   private observableSubscription?: plugins.smartrx.rxjs.Subscription;
   private debugLog(messageArg: string) {
@@ -46,24 +63,35 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   }
 
   constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
-    super(optionsArg);
+    super(Object.assign({
+      highWaterMark: 1,
+    }, optionsArg));
     this.options = optionsArg;
+    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(this.options.highWaterMark || 1)
   }
 
   public async _read(size: number): Promise<void> {
-    await this.backpressuredArray.waitForItems();
     this.debugLog(`${this.options.name}: read was called`);
+    await this.backpressuredArray.waitForItems();
+    this.debugLog(`${this.options.name}: successfully waited for items.`);
     if (this.options.readFunction) {
       await this.options.readFunction();
     }
     let canPushMore = true; 
     while(this.backpressuredArray.data.length > 0 && canPushMore) {
       const nextChunk = this.backpressuredArray.shift();
-      if (nextChunk) {
       canPushMore = this.push(nextChunk);
     }
   }
+
+  public async backpressuredPush (pushArg: TOutput) {
+    const canPushMore = this.backpressuredArray.push(pushArg);
+    if (!canPushMore) {
+      this.debugLog(`${this.options.name}: cannot push more`);
+      await this.backpressuredArray.waitForSpace();
+      this.debugLog(`${this.options.name}: can push more again`);
     }
+  };
 
   private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
   // Ensure the _write method types the chunk as TInput and encodes TOutput
@@ -79,9 +107,9 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
         isTruncated = true;
         callback();
       },
-      push: (pushArg: TOutput) => {
+      push: async (pushArg: TOutput) => {
-        this.backpressuredArray.push(pushArg);
+        await this.backpressuredPush(pushArg);
-      },
+      }
     };
 
     try {
@@ -92,12 +120,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
         return;
       }
       if (modifiedChunk) {
-        const canPushMore = this.backpressuredArray.push(modifiedChunk);
+        await tools.push(modifiedChunk);
-        if (!canPushMore) {
-          this.debugLog(`${this.options.name}: cannot push more`);
-          await this.backpressuredArray.waitForSpace();
-          this.debugLog(`${this.options.name}: can push more again`);
-        }
       }
       callback();
       writeDeferred.resolve();
@@ -114,21 +137,23 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     if (this.options.finalFunction) {
       const tools: IStreamTools = {
         truncate: () => callback(),
-        push: (pipeObject) => this.push(pipeObject),
+        push: async (pipeObject) => {
+          this.backpressuredArray.push(pipeObject);
+        },
       };
 
       try {
         const finalChunk = await this.options.finalFunction(tools);
         if (finalChunk) {
-          this.push(finalChunk);
+          this.backpressuredArray.push(finalChunk);
         }
       } catch (err) {
-        this.push(null);
+        this.backpressuredArray.push(null);
         callback(err);
         return;
       }
     }
-    this.push(null);
+    this.backpressuredArray.push(null);
     callback();
   }
 }

ts/smartstream.functions.ts

@@ -19,3 +19,12 @@ export function createTransformFunction<TInput, TOutput>(
 
   return smartDuplexStream;
 }
+
+export const createPassThrough = () => {
+  return new SmartDuplex({
+    objectMode: true,
+    writeFunction: async (chunkArg, toolsArg) => {
+      return chunkArg;
+    }
+  })
+}