3.1.0

changelog.md

@@ -0,0 +1,47 @@
+# Changelog
+
+## 2024-10-13 - 3.1.0 - feat(core)
+Add support for creating Web ReadableStream from a file
+
+- Introduced a new helper function `createWebReadableStreamFromFile` that allows for creating a Web ReadableStream from a file path.
+- Updated exports in `ts/index.ts` to include `nodewebhelpers` which provides the new web stream feature.
+
+## 2024-10-13 - 3.0.46 - fix(WebDuplexStream)
+Fix errors in WebDuplexStream transformation and test logic
+
+- Corrected async handling in WebDuplexStream write function
+- Fixed `WebDuplexStream` tests to properly handle asynchronous reading and writing
+
+## 2024-10-13 - 3.0.45 - fix(ts)
+Fixed formatting issues in SmartDuplex class
+
+- Resolved inconsistent spacing in SmartDuplex class methods and constructor.
+- Ensured consistent formatting in the getWebStreams method.
+
+## 2024-06-02 - 3.0.39 - smartduplex
+Add .getWebStreams method
+
+- Introduced a new `.getWebStreams` method in the smartduplex module, providing compatibility with the web streams API.
+
+## 2024-03-16 - 3.0.34 - configuration
+Update project configuration files
+
+- Updated `tsconfig` for optimization.
+- Modified `npmextra.json` to set the `githost` attribute.
+
+## 2023-11-03 - 3.0.0 to 3.0.8 - core
+Transition to major version 3.x
+
+- Implemented breaking changes in the core system for better performance and feature set.
+- Continuous core updates to improve stability and performance across minor version increments.
+  
+## 2023-11-02 - 2.0.4 to 2.0.8 - core
+Core updates and a major fix
+
+- Implemented core updates addressing minor bugs and enhancements.
+- A significant breaking change update transitioning from 2.0.x to 3.0.0.
+  
+## 2022-03-31 - 2.0.0 - core
+Major esm transition
+
+- Implemented a breaking change by switching the core to ESM (ECMAScript Module) format for modernized module handling.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "3.0.44",
+  "version": "3.1.0",
   "private": false,
   "description": "A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.",
   "type": "module",

test/test.ts_web.both.ts

@@ -1,37 +1,37 @@
-import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import { expect, tap } from '@push.rocks/tapbundle';
 import * as webstream from '../ts_web/index.js';
 
-tap.test('WebDuplexStream', async (toolsArg) => {
+tap.test('WebDuplexStream fromUInt8Array should read back the same Uint8Array', async () => {
-  const testDone = toolsArg.defer(); // Create a deferred object to control test completion.
   const inputUint8Array = new Uint8Array([1, 2, 3, 4, 5]);
   const stream = webstream.WebDuplexStream.fromUInt8Array(inputUint8Array);
 
   const reader = stream.readable.getReader();
   let readUint8Array = new Uint8Array();
 
-  reader.read().then(function processText({ done, value }) {
+  // Read from the stream
-    if (done) {
+  while (true) {
-      expect(readUint8Array).toEqual(inputUint8Array);
+    const { value, done } = await reader.read();
-      testDone.resolve(); // Correctly signal that the test is done.
+    if (done) break;
-      return;
+    if (value) {
+      // Concatenate value to readUint8Array
+      const tempArray = new Uint8Array(readUint8Array.length + value.length);
+      tempArray.set(readUint8Array, 0);
+      tempArray.set(value, readUint8Array.length);
+      readUint8Array = tempArray;
     }
-    readUint8Array = new Uint8Array([...readUint8Array, ...value]);
+  }
-    return reader.read().then(processText);
-  });
 
-  return testDone.promise; // Return the promise to properly wait for the test to complete.
+  expect(readUint8Array).toEqual(inputUint8Array);
 });
 
-
+tap.test('WebDuplexStream should handle transform with a write function', async () => {
-tap.test('should handle transform with a write function', async (toolsArg) => {
-  const testDone = toolsArg.defer();
   const input = [1, 2, 3, 4, 5];
   const expectedOutput = [2, 4, 6, 8, 10];
 
   const transformStream = new webstream.WebDuplexStream<number, number>({
-    writeFunction: (chunk, { push }) => {
+    writeFunction: async (chunk, { push }) => {
-      push(chunk * 2); // Push the doubled number into the stream
+      // Push the doubled number into the stream
-      return Promise.resolve(); // Resolve the promise immediately
+      push(chunk * 2);
     },
   });
 
@@ -40,31 +40,28 @@ tap.test('should handle transform with a write function', async (toolsArg) => {
 
   const output: number[] = [];
 
-  // Process the text and resolve the test once done.
+  // Read from the stream asynchronously
-  const processText = async ({ done, value }) => {
+  const readPromise = (async () => {
-    if (done) {
+    while (true) {
-      expect(output).toEqual(expectedOutput);
+      const { value, done } = await readableStream.read();
-      testDone.resolve(); // Resolve the deferred test once all values have been read.
+      if (done) break;
-      return;
+      if (value !== undefined) {
+        output.push(value);
+      }
     }
-    if (value !== undefined) {
+  })();
-      output.push(value);
-    }
-    // Continue reading and processing.
-    await readableStream.read().then(processText);
-  };
 
-  // Start the read process before writing to the stream.
+  // Write to the stream
-  readableStream.read().then(processText);
-
-  // Sequentially write to the stream and close when done.
   for (const num of input) {
     await writableStream.write(num);
   }
   await writableStream.close();
 
-  return testDone.promise; // This will wait until the testDone is resolved before completing the test.
+  // Wait for the reading to complete
+  await readPromise;
+
+  // Assert that the output matches the expected transformed data
+  expect(output).toEqual(expectedOutput);
 });
 
-
+tap.start();
-tap.start();

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.44',
+  version: '3.1.0',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }

ts/index.ts

@@ -11,3 +11,7 @@ export * from './smartstream.functions.js';
 
 import * as plugins from './smartstream.plugins.js';
 export const webstream = plugins.webstream;
+import * as nodewebhelpers from './smartstream.nodewebhelpers.js';
+export {
+  nodewebhelpers,
+}

ts/smartstream.classes.smartduplex.ts

@@ -34,9 +34,9 @@ export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
    * it can push or return chunks (but does not have to) to be written to the readable side of the stream
    */
   writeFunction?: IStreamWriteFunction<TInput, TOutput>;
-  
+
   /**
-   * a final function that is run at the end of the stream 
+   * a final function that is run at the end of the stream
    */
   finalFunction?: IStreamFinalFunction<TOutput>;
 }
@@ -53,21 +53,29 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   }
 
   // INSTANCE
-  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
+  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>; // an array that only takes a defined amount of items
   public options: ISmartDuplexOptions<TInput, TOutput>;
   private observableSubscription?: plugins.smartrx.rxjs.Subscription;
   private debugLog(messageArg: string) {
+    // optional debug log
     if (this.options.debug) {
       console.log(messageArg);
     }
   }
 
   constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
-    super(Object.assign({
+    super(
-      highWaterMark: 1,
+      Object.assign(
-    }, optionsArg));
+        {
+          highWaterMark: 1,
+        },
+        optionsArg
+      )
+    );
     this.options = optionsArg;
-    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(this.options.highWaterMark || 1)
+    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(
+      this.options.highWaterMark || 1
+    );
   }
 
   public async _read(size: number): Promise<void> {
@@ -77,14 +85,14 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     if (this.options.readFunction) {
       await this.options.readFunction();
     }
-    let canPushMore = true; 
+    let canPushMore = true;
-    while(this.backpressuredArray.data.length > 0 && canPushMore) {
+    while (this.backpressuredArray.data.length > 0 && canPushMore) {
       const nextChunk = this.backpressuredArray.shift();
       canPushMore = this.push(nextChunk);
     }
   }
 
-  public async backpressuredPush (pushArg: TOutput) {
+  public async backpressuredPush(pushArg: TOutput) {
     const canPushMore = this.backpressuredArray.push(pushArg);
     if (!canPushMore) {
       this.debugLog(`${this.options.name}: cannot push more`);
@@ -92,7 +100,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
       this.debugLog(`${this.options.name}: can push more again`);
     }
     return canPushMore;
-  };
+  }
 
   private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
   // Ensure the _write method types the chunk as TInput and encodes TOutput
@@ -110,7 +118,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
       },
       push: async (pushArg: TOutput) => {
         return await this.backpressuredPush(pushArg);
-      }
+      },
     };
 
     try {
@@ -158,7 +166,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     callback();
   }
 
-  public async getWebStreams(): Promise<{ readable: ReadableStream, writable: WritableStream }> {
+  public async getWebStreams(): Promise<{ readable: ReadableStream; writable: WritableStream }> {
     const duplex = this;
     const readable = new ReadableStream({
       start(controller) {
@@ -168,16 +176,16 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
             controller.enqueue(chunk);
           }
         });
-  
+
         duplex.on('end', () => {
           controller.close();
         });
       },
       cancel(reason) {
         duplex.destroy(new Error(reason));
-      }
+      },
     });
-  
+
     const writable = new WritableStream({
       write(chunk) {
         return new Promise<void>((resolve, reject) => {
@@ -188,7 +196,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
               resolve();
             }
           });
-  
+
           if (isBackpressured) {
             duplex.once('drain', resolve);
           }
@@ -201,9 +209,9 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
       },
       abort(reason) {
         duplex.destroy(new Error(reason));
-      }
+      },
     });
-  
+
     return { readable, writable };
   }
 }

ts/smartstream.nodewebhelpers.ts

@@ -0,0 +1,34 @@
+import { createReadStream } from 'fs';
+
+/**
+ * Creates a Web ReadableStream from a file.
+ *
+ * @param filePath - The path to the file to be read
+ * @returns A Web ReadableStream that reads the file in chunks
+ */
+export function createWebReadableStreamFromFile(filePath: string): ReadableStream<Uint8Array> {
+  const fileStream = createReadStream(filePath);
+
+  return new ReadableStream({
+    start(controller) {
+      // When data is available, enqueue it into the Web ReadableStream
+      fileStream.on('data', (chunk) => {
+        controller.enqueue(chunk as Uint8Array);
+      });
+
+      // When the file stream ends, close the Web ReadableStream
+      fileStream.on('end', () => {
+        controller.close();
+      });
+
+      // If there's an error, error the Web ReadableStream
+      fileStream.on('error', (err) => {
+        controller.error(err);
+      });
+    },
+    cancel() {
+      // If the Web ReadableStream is canceled, destroy the file stream
+      fileStream.destroy();
+    }
+  });
+}

ts_web/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.44',
+  version: '3.1.0',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }

ts_web/classes.webduplexstream.ts

@@ -63,6 +63,7 @@ export class WebDuplexStream<TInput = any, TOutput = any> extends TransformStrea
   options: WebDuplexStreamOptions<TInput, TOutput>;
 
   constructor(optionsArg: WebDuplexStreamOptions<TInput, TOutput>) {
+    // here we call into the official web stream api
     super({
       async transform(chunk, controller) {
         // Transformation logic remains unchanged
@@ -72,15 +73,14 @@ export class WebDuplexStream<TInput = any, TOutput = any> extends TransformStrea
             push: (pushArg: TOutput) => controller.enqueue(pushArg),
           };
         
-          optionsArg.writeFunction(chunk, tools)
+          try {
-            .then(writeReturnChunk => {
+            const writeReturnChunk = await optionsArg.writeFunction(chunk, tools);
-              // the write return chunk is optional
+            if (writeReturnChunk) { // return chunk is optional
-              // just in case the write function returns something other than void.
+              controller.enqueue(writeReturnChunk);
-              if (writeReturnChunk) {
+            }
-                controller.enqueue(writeReturnChunk);
+          } catch (err) {
-              }
+            controller.error(err);
-            })
+          }
-            .catch(err => controller.error(err));
         } else {
           controller.error(new Error('No write function provided'));
         }