3.1.1

changelog.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 2024-10-13 - 3.1.1 - fix(WebDuplexStream)
+Improved read/write interface and error handling in WebDuplexStream
+
+- Enhanced the IStreamToolsRead and IStreamToolsWrite interfaces for better Promise handling
+- Refined readFunction and writeFunction handling to accommodate asynchronous data processing and error propagation
+- Added internal _startReading method to facilitate initial data handling if readFunction is present
+- Maintained backward compatibility while ensuring data continuity when no writeFunction is specified
+
+## 2024-10-13 - 3.1.0 - feat(core)
+Add support for creating Web ReadableStream from a file
+
+- Introduced a new helper function `createWebReadableStreamFromFile` that allows for creating a Web ReadableStream from a file path.
+- Updated exports in `ts/index.ts` to include `nodewebhelpers` which provides the new web stream feature.
+
+## 2024-10-13 - 3.0.46 - fix(WebDuplexStream)
+Fix errors in WebDuplexStream transformation and test logic
+
+- Corrected async handling in WebDuplexStream write function
+- Fixed `WebDuplexStream` tests to properly handle asynchronous reading and writing
+
 ## 2024-10-13 - 3.0.45 - fix(ts)
 Fixed formatting issues in SmartDuplex class
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "3.0.45",
+  "version": "3.1.1",
   "private": false,
   "description": "A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.",
   "type": "module",

test/test.ts_web.both.ts

@@ -1,37 +1,37 @@
-import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import { expect, tap } from '@push.rocks/tapbundle';
 import * as webstream from '../ts_web/index.js';
 
-tap.test('WebDuplexStream', async (toolsArg) => {
+tap.test('WebDuplexStream fromUInt8Array should read back the same Uint8Array', async () => {
-  const testDone = toolsArg.defer(); // Create a deferred object to control test completion.
   const inputUint8Array = new Uint8Array([1, 2, 3, 4, 5]);
   const stream = webstream.WebDuplexStream.fromUInt8Array(inputUint8Array);
 
   const reader = stream.readable.getReader();
   let readUint8Array = new Uint8Array();
 
-  reader.read().then(function processText({ done, value }) {
+  // Read from the stream
-    if (done) {
+  while (true) {
-      expect(readUint8Array).toEqual(inputUint8Array);
+    const { value, done } = await reader.read();
-      testDone.resolve(); // Correctly signal that the test is done.
+    if (done) break;
-      return;
+    if (value) {
+      // Concatenate value to readUint8Array
+      const tempArray = new Uint8Array(readUint8Array.length + value.length);
+      tempArray.set(readUint8Array, 0);
+      tempArray.set(value, readUint8Array.length);
+      readUint8Array = tempArray;
     }
-    readUint8Array = new Uint8Array([...readUint8Array, ...value]);
+  }
-    return reader.read().then(processText);
+
+  expect(readUint8Array).toEqual(inputUint8Array);
 });
 
-  return testDone.promise; // Return the promise to properly wait for the test to complete.
+tap.test('WebDuplexStream should handle transform with a write function', async () => {
-});
-
-
-tap.test('should handle transform with a write function', async (toolsArg) => {
-  const testDone = toolsArg.defer();
   const input = [1, 2, 3, 4, 5];
   const expectedOutput = [2, 4, 6, 8, 10];
 
   const transformStream = new webstream.WebDuplexStream<number, number>({
-    writeFunction: (chunk, { push }) => {
+    writeFunction: async (chunk, { push }) => {
-      push(chunk * 2); // Push the doubled number into the stream
+      // Push the doubled number into the stream
-      return Promise.resolve(); // Resolve the promise immediately
+      push(chunk * 2);
     },
   });
 
@@ -40,31 +40,28 @@ tap.test('should handle transform with a write function', async (toolsArg) => {
 
   const output: number[] = [];
 
-  // Process the text and resolve the test once done.
+  // Read from the stream asynchronously
-  const processText = async ({ done, value }) => {
+  const readPromise = (async () => {
-    if (done) {
+    while (true) {
-      expect(output).toEqual(expectedOutput);
+      const { value, done } = await readableStream.read();
-      testDone.resolve(); // Resolve the deferred test once all values have been read.
+      if (done) break;
-      return;
-    }
       if (value !== undefined) {
         output.push(value);
       }
-    // Continue reading and processing.
+    }
-    await readableStream.read().then(processText);
+  })();
-  };
 
-  // Start the read process before writing to the stream.
+  // Write to the stream
-  readableStream.read().then(processText);
-
-  // Sequentially write to the stream and close when done.
   for (const num of input) {
     await writableStream.write(num);
   }
   await writableStream.close();
 
-  return testDone.promise; // This will wait until the testDone is resolved before completing the test.
+  // Wait for the reading to complete
+  await readPromise;
+
+  // Assert that the output matches the expected transformed data
+  expect(output).toEqual(expectedOutput);
 });
 
-
 tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.45',
+  version: '3.1.1',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }

ts/index.ts

@@ -11,3 +11,7 @@ export * from './smartstream.functions.js';
 
 import * as plugins from './smartstream.plugins.js';
 export const webstream = plugins.webstream;
+import * as nodewebhelpers from './smartstream.nodewebhelpers.js';
+export {
+  nodewebhelpers,
+}

ts/smartstream.nodewebhelpers.ts

@@ -0,0 +1,34 @@
+import { createReadStream } from 'fs';
+
+/**
+ * Creates a Web ReadableStream from a file.
+ *
+ * @param filePath - The path to the file to be read
+ * @returns A Web ReadableStream that reads the file in chunks
+ */
+export function createWebReadableStreamFromFile(filePath: string): ReadableStream<Uint8Array> {
+  const fileStream = createReadStream(filePath);
+
+  return new ReadableStream({
+    start(controller) {
+      // When data is available, enqueue it into the Web ReadableStream
+      fileStream.on('data', (chunk) => {
+        controller.enqueue(chunk as Uint8Array);
+      });
+
+      // When the file stream ends, close the Web ReadableStream
+      fileStream.on('end', () => {
+        controller.close();
+      });
+
+      // If there's an error, error the Web ReadableStream
+      fileStream.on('error', (err) => {
+        controller.error(err);
+      });
+    },
+    cancel() {
+      // If the Web ReadableStream is canceled, destroy the file stream
+      fileStream.destroy();
+    }
+  });
+}

ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.45',
+  version: '3.1.1',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }

ts_web/classes.webduplexstream.ts

@@ -1,25 +1,22 @@
 import * as plugins from './plugins.js';
 
-
 // ========================================
-// READ
+// Interfaces for Read functionality
 // ========================================
 export interface IStreamToolsRead<TInput, TOutput> {
   done: () => void;
-  write: (writeArg: TInput) => void;
+  write: (writeArg: TInput) => Promise<void>;
 }
 
 /**
- * the read function is called anytime
+ * The read function is called when data needs to be read into the stream.
- *   -> the WebDuplexStream is being read from
- *      and at the same time if nothing is enqueued
  */
 export interface IStreamReadFunction<TInput, TOutput> {
   (toolsArg: IStreamToolsRead<TInput, TOutput>): Promise<void>;
 }
 
 // ========================================
-// WRITE
+// Interfaces for Write functionality
 // ========================================
 export interface IStreamToolsWrite<TInput, TOutput> {
   truncate: () => void;
@@ -27,15 +24,14 @@ export interface IStreamToolsWrite<TInput, TOutput> {
 }
 
 /**
- * the write function can return something.
+ * The write function is called whenever a chunk is written to the stream.
- * It is called anytime a chunk is written to the stream.
  */
 export interface IStreamWriteFunction<TInput, TOutput> {
   (chunkArg: TInput, toolsArg: IStreamToolsWrite<TInput, TOutput>): Promise<any>;
 }
 
 export interface IStreamFinalFunction<TInput, TOutput> {
-  (toolsArg: IStreamToolsWrite<TInput, TOutput>): Promise<TOutput>;
+  (toolsArg: IStreamToolsWrite<TInput, TOutput>): Promise<TOutput | void>;
 }
 
 export interface WebDuplexStreamOptions<TInput, TOutput> {
@@ -45,12 +41,90 @@ export interface WebDuplexStreamOptions<TInput, TOutput> {
 }
 
 export class WebDuplexStream<TInput = any, TOutput = any> extends TransformStream<TInput, TOutput> {
+  // INSTANCE
+  options: WebDuplexStreamOptions<TInput, TOutput>;
+
+  constructor(optionsArg: WebDuplexStreamOptions<TInput, TOutput>) {
+    super({
+      async start(controller) {
+        // Optionally initialize any state here
+      },
+      async transform(chunk, controller) {
+        if (optionsArg?.writeFunction) {
+          const tools: IStreamToolsWrite<TInput, TOutput> = {
+            truncate: () => controller.terminate(),
+            push: (pushArg: TOutput) => controller.enqueue(pushArg),
+          };
+
+          try {
+            const writeReturnChunk = await optionsArg.writeFunction(chunk, tools);
+            if (writeReturnChunk !== undefined && writeReturnChunk !== null) {
+              controller.enqueue(writeReturnChunk);
+            }
+          } catch (err) {
+            controller.error(err);
+          }
+        } else {
+          // If no writeFunction is provided, pass the chunk through
+          controller.enqueue(chunk as unknown as TOutput);
+        }
+      },
+      async flush(controller) {
+        if (optionsArg?.finalFunction) {
+          const tools: IStreamToolsWrite<TInput, TOutput> = {
+            truncate: () => controller.terminate(),
+            push: (pushArg) => controller.enqueue(pushArg),
+          };
+
+          try {
+            const finalChunk = await optionsArg.finalFunction(tools);
+            if (finalChunk) {
+              controller.enqueue(finalChunk);
+            }
+          } catch (err) {
+            controller.error(err);
+          } finally {
+            controller.terminate();
+          }
+        } else {
+          controller.terminate();
+        }
+      },
+    });
+
+    this.options = optionsArg;
+
+    // Start producing data if readFunction is provided
+    if (this.options.readFunction) {
+      this._startReading();
+    }
+  }
+
+  private async _startReading() {
+    const writable = this.writable;
+    const writer = writable.getWriter();
+
+    const tools: IStreamToolsRead<TInput, TOutput> = {
+      done: () => writer.close(),
+      write: async (writeArg) => await writer.write(writeArg),
+    };
+
+    try {
+      await this.options.readFunction(tools);
+    } catch (err) {
+      writer.abort(err);
+    } finally {
+      writer.releaseLock();
+    }
+  }
+
+  // Static method example (adjust as needed)
   static fromUInt8Array(uint8Array: Uint8Array): WebDuplexStream<Uint8Array, Uint8Array> {
     const stream = new WebDuplexStream<Uint8Array, Uint8Array>({
       writeFunction: async (chunk, { push }) => {
         push(chunk); // Directly push the chunk as is
         return null;
-      }
+      },
     });
 
     const writer = stream.writable.getWriter();
@@ -58,99 +132,4 @@ export class WebDuplexStream<TInput = any, TOutput = any> extends TransformStrea
 
     return stream;
   }
-  
-  // INSTANCE
-  options: WebDuplexStreamOptions<TInput, TOutput>;
-
-  constructor(optionsArg: WebDuplexStreamOptions<TInput, TOutput>) {
-    super({
-      async transform(chunk, controller) {
-        // Transformation logic remains unchanged
-        if (optionsArg?.writeFunction) {
-          const tools: IStreamToolsWrite<TInput, TOutput> = {
-            truncate: () => controller.terminate(),
-            push: (pushArg: TOutput) => controller.enqueue(pushArg),
-          };
-        
-          optionsArg.writeFunction(chunk, tools)
-            .then(writeReturnChunk => {
-              // the write return chunk is optional
-              // just in case the write function returns something other than void.
-              if (writeReturnChunk) {
-                controller.enqueue(writeReturnChunk);
-              }
-            })
-            .catch(err => controller.error(err));
-        } else {
-          controller.error(new Error('No write function provided'));
-        }
-      },
-      async flush(controller) {
-        // Flush logic remains unchanged
-        if (optionsArg?.finalFunction) {
-          const tools: IStreamToolsWrite<TInput, TOutput> = {
-            truncate: () => controller.terminate(),
-            push: (pipeObject) => controller.enqueue(pipeObject),
-          };
-      
-          optionsArg.finalFunction(tools)
-            .then(finalChunk => {
-              if (finalChunk) {
-                controller.enqueue(finalChunk);
-              }
-            })  
-            .catch(err => controller.error(err))
-            .finally(() => controller.terminate());
-        } else {
-          controller.terminate();
-        }
-      }
-    });
-
-    this.options = optionsArg;
-  }
-
-  // Method to create a custom readable stream that integrates the readFunction
-  // readFunction is executed whenever the stream is being read from and nothing is enqueued
-  getCustomReadableStream() {
-    const readableStream = this.readable;
-    const options = this.options;
-    const customReadable = new ReadableStream({
-      async pull(controller) {
-        const reader = readableStream.getReader();
-
-        // Check the current state of the original stream
-        const { value, done } = await reader.read();
-        reader.releaseLock();
-
-        if (done) {
-          // If the original stream is done, close the custom readable stream
-          controller.close();
-        } else {
-          if (value) {
-            // If there is data in the original stream, enqueue it and do not execute the readFunction
-            controller.enqueue(value);
-          } else if (options.readFunction) {
-            // If the original stream is empty, execute the readFunction and read again
-            await options.readFunction({
-              done: () => controller.close(),
-              write: (writeArg) => controller.enqueue(writeArg),
-            });
-            
-            const newReader = readableStream.getReader();
-            const { value: newValue, done: newDone } = await newReader.read();
-            newReader.releaseLock();
-
-            if (newDone) {
-              controller.close();
-            } else {
-              controller.enqueue(newValue);
-            }
-          }
-        }
-      }
-    });
-
-    return customReadable;
-  }
 }