3.0.45

.gitlab-ci.yml

@@ -1,140 +0,0 @@
-# gitzone ci_default
-image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-
-cache:
-  paths:
-    - .npmci_cache/
-  key: '$CI_BUILD_STAGE'
-
-stages:
-  - security
-  - test
-  - release
-  - metadata
-
-before_script:
-  - npm install -g @shipzone/npmci
-
-# ====================
-# security stage
-# ====================
-mirror:
-  stage: security
-  script:
-    - npmci git mirror
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-auditProductionDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --production --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=prod --production
-  tags:
-    - docker
-  allow_failure: true
-
-auditDevDependencies:
-  image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
-  stage: security
-  script:
-    - npmci npm prepare
-    - npmci command npm install --ignore-scripts
-    - npmci command npm config set registry https://registry.npmjs.org
-    - npmci command npm audit --audit-level=high --only=dev
-  tags:
-    - docker
-  allow_failure: true
-
-# ====================
-# test stage
-# ====================
-
-testStable:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci npm test
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-testBuild:
-  stage: test
-  script:
-    - npmci npm prepare
-    - npmci node install stable
-    - npmci npm install
-    - npmci command npm run build
-  coverage: /\d+.?\d+?\%\s*coverage/
-  tags:
-    - docker
-
-release:
-  stage: release
-  script:
-    - npmci node install stable
-    - npmci npm publish
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-# ====================
-# metadata stage
-# ====================
-codequality:
-  stage: metadata
-  allow_failure: true
-  only:
-    - tags
-  script:
-    - npmci command npm install -g typescript
-    - npmci npm prepare
-    - npmci npm install
-  tags:
-    - lossless
-    - docker
-    - priv
-
-trigger:
-  stage: metadata
-  script:
-    - npmci trigger
-  only:
-    - tags
-  tags:
-    - lossless
-    - docker
-    - notpriv
-
-pages:
-  stage: metadata
-  script:
-    - npmci node install lts
-    - npmci command npm install -g @git.zone/tsdoc
-    - npmci npm prepare
-    - npmci npm install
-    - npmci command tsdoc
-  tags:
-    - lossless
-    - docker
-    - notpriv
-  only:
-    - tags
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - public
-  allow_failure: true

assets/denoentry.ts

@@ -1 +0,0 @@
-console.log('Hello from deno');

changelog.md

@@ -0,0 +1,35 @@
+# Changelog
+
+## 2024-10-13 - 3.0.45 - fix(ts)
+Fixed formatting issues in SmartDuplex class
+
+- Resolved inconsistent spacing in SmartDuplex class methods and constructor.
+- Ensured consistent formatting in the getWebStreams method.
+
+## 2024-06-02 - 3.0.39 - smartduplex
+Add .getWebStreams method
+
+- Introduced a new `.getWebStreams` method in the smartduplex module, providing compatibility with the web streams API.
+
+## 2024-03-16 - 3.0.34 - configuration
+Update project configuration files
+
+- Updated `tsconfig` for optimization.
+- Modified `npmextra.json` to set the `githost` attribute.
+
+## 2023-11-03 - 3.0.0 to 3.0.8 - core
+Transition to major version 3.x
+
+- Implemented breaking changes in the core system for better performance and feature set.
+- Continuous core updates to improve stability and performance across minor version increments.
+  
+## 2023-11-02 - 2.0.4 to 2.0.8 - core
+Core updates and a major fix
+
+- Implemented core updates addressing minor bugs and enhancements.
+- A significant breaking change update transitioning from 2.0.x to 3.0.0.
+  
+## 2022-03-31 - 2.0.0 - core
+Major esm transition
+
+- Implemented a breaking change by switching the core to ESM (ECMAScript Module) format for modernized module handling.

package.json

@@ -1,39 +1,40 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "3.0.36",
+  "version": "3.0.45",
   "private": false,
   "description": "A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.",
-  "main": "dist_ts/index.js",
-  "typings": "dist_ts/index.d.ts",
   "type": "module",
+  "exports": {
+    ".": "./dist_ts/index.js",
+    "./web": "./dist_ts_web/index.js"
+  },
   "scripts": {
     "test": "(tstest test/)",
-    "build": "(tsbuild)",
-    "buildDocs": "tsdoc"
+    "build": "(tsbuild tsfolders --web --allowimplicitany)"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://gitlab.com/push.rocks/smartstream.git"
+    "url": "https://code.foss.global/push.rocks/smartstream.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
     "url": "https://gitlab.com/push.rocks/smartstream/issues"
   },
-  "homepage": "https://gitlab.com/push.rocks/smartstream#readme",
+  "homepage": "https://code.foss.global/push.rocks/smartstream",
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.72",
+    "@git.zone/tsbuild": "^2.1.80",
     "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.88",
-    "@push.rocks/smartfile": "^11.0.4",
-    "@push.rocks/tapbundle": "^5.0.17",
-    "@types/node": "^20.11.28"
+    "@git.zone/tstest": "^1.0.90",
+    "@push.rocks/smartfile": "^11.0.15",
+    "@push.rocks/tapbundle": "^5.0.23",
+    "@types/node": "^20.12.12"
   },
   "dependencies": {
-    "@push.rocks/lik": "^6.0.14",
+    "@push.rocks/lik": "^6.0.15",
+    "@push.rocks/smartenv": "^5.0.12",
     "@push.rocks/smartpromise": "^4.0.3",
-    "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/webstream": "^1.0.8"
+    "@push.rocks/smartrx": "^3.0.7"
   },
   "browserslist": [
     "last 1 chrome versions"

test/test.ts_web.both.ts

@@ -0,0 +1,70 @@
+import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import * as webstream from '../ts_web/index.js';
+
+tap.test('WebDuplexStream', async (toolsArg) => {
+  const testDone = toolsArg.defer(); // Create a deferred object to control test completion.
+  const inputUint8Array = new Uint8Array([1, 2, 3, 4, 5]);
+  const stream = webstream.WebDuplexStream.fromUInt8Array(inputUint8Array);
+
+  const reader = stream.readable.getReader();
+  let readUint8Array = new Uint8Array();
+
+  reader.read().then(function processText({ done, value }) {
+    if (done) {
+      expect(readUint8Array).toEqual(inputUint8Array);
+      testDone.resolve(); // Correctly signal that the test is done.
+      return;
+    }
+    readUint8Array = new Uint8Array([...readUint8Array, ...value]);
+    return reader.read().then(processText);
+  });
+
+  return testDone.promise; // Return the promise to properly wait for the test to complete.
+});
+
+
+tap.test('should handle transform with a write function', async (toolsArg) => {
+  const testDone = toolsArg.defer();
+  const input = [1, 2, 3, 4, 5];
+  const expectedOutput = [2, 4, 6, 8, 10];
+
+  const transformStream = new webstream.WebDuplexStream<number, number>({
+    writeFunction: (chunk, { push }) => {
+      push(chunk * 2); // Push the doubled number into the stream
+      return Promise.resolve(); // Resolve the promise immediately
+    },
+  });
+
+  const writableStream = transformStream.writable.getWriter();
+  const readableStream = transformStream.readable.getReader();
+
+  const output: number[] = [];
+
+  // Process the text and resolve the test once done.
+  const processText = async ({ done, value }) => {
+    if (done) {
+      expect(output).toEqual(expectedOutput);
+      testDone.resolve(); // Resolve the deferred test once all values have been read.
+      return;
+    }
+    if (value !== undefined) {
+      output.push(value);
+    }
+    // Continue reading and processing.
+    await readableStream.read().then(processText);
+  };
+
+  // Start the read process before writing to the stream.
+  readableStream.read().then(processText);
+
+  // Sequentially write to the stream and close when done.
+  for (const num of input) {
+    await writableStream.write(num);
+  }
+  await writableStream.close();
+
+  return testDone.promise; // This will wait until the testDone is resolved before completing the test.
+});
+
+
+tap.start();

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.36',
+  version: '3.0.45',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }

ts/index.ts

@@ -1,3 +1,8 @@
+import { stream } from './smartstream.plugins.js';
+export {
+  stream,
+}
+
 export * from './smartstream.classes.smartduplex.js';
 export * from './smartstream.classes.streamwrapper.js';
 export * from './smartstream.classes.streamintake.js';

ts/smartstream.classes.smartduplex.ts

@@ -53,21 +53,29 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   }
 
   // INSTANCE
-  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
+  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>; // an array that only takes a defined amount of items
   public options: ISmartDuplexOptions<TInput, TOutput>;
   private observableSubscription?: plugins.smartrx.rxjs.Subscription;
   private debugLog(messageArg: string) {
+    // optional debug log
     if (this.options.debug) {
       console.log(messageArg);
     }
   }
 
   constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
-    super(Object.assign({
+    super(
+      Object.assign(
+        {
           highWaterMark: 1,
-    }, optionsArg));
+        },
+        optionsArg
+      )
+    );
     this.options = optionsArg;
-    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(this.options.highWaterMark || 1)
+    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(
+      this.options.highWaterMark || 1
+    );
   }
 
   public async _read(size: number): Promise<void> {
@@ -92,7 +100,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
       this.debugLog(`${this.options.name}: can push more again`);
     }
     return canPushMore;
-  };
+  }
 
   private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
   // Ensure the _write method types the chunk as TInput and encodes TOutput
@@ -110,7 +118,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
       },
       push: async (pushArg: TOutput) => {
         return await this.backpressuredPush(pushArg);
-      }
+      },
     };
 
     try {
@@ -157,4 +165,53 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     this.backpressuredArray.push(null);
     callback();
   }
+
+  public async getWebStreams(): Promise<{ readable: ReadableStream; writable: WritableStream }> {
+    const duplex = this;
+    const readable = new ReadableStream({
+      start(controller) {
+        duplex.on('readable', () => {
+          let chunk;
+          while (null !== (chunk = duplex.read())) {
+            controller.enqueue(chunk);
+          }
+        });
+
+        duplex.on('end', () => {
+          controller.close();
+        });
+      },
+      cancel(reason) {
+        duplex.destroy(new Error(reason));
+      },
+    });
+
+    const writable = new WritableStream({
+      write(chunk) {
+        return new Promise<void>((resolve, reject) => {
+          const isBackpressured = !duplex.write(chunk, (error) => {
+            if (error) {
+              reject(error);
+            } else {
+              resolve();
+            }
+          });
+
+          if (isBackpressured) {
+            duplex.once('drain', resolve);
+          }
+        });
+      },
+      close() {
+        return new Promise<void>((resolve, reject) => {
+          duplex.end(resolve);
+        });
+      },
+      abort(reason) {
+        duplex.destroy(new Error(reason));
+      },
+    });
+
+    return { readable, writable };
+  }
 }

ts/smartstream.classes.streamintake.ts

@@ -1,6 +1,45 @@
 import * as plugins from './smartstream.plugins.js';
 
 export class StreamIntake<T> extends plugins.stream.Readable {
+  // STATIC
+  public static async fromStream<U>(inputStream: plugins.stream.Readable | ReadableStream, options?: plugins.stream.ReadableOptions): Promise<StreamIntake<U>> {
+    const intakeStream = new StreamIntake<U>(options);
+
+    if (inputStream instanceof plugins.stream.Readable) {
+      inputStream.on('data', (chunk: U) => {
+        intakeStream.pushData(chunk);
+      });
+
+      inputStream.on('end', () => {
+        intakeStream.signalEnd();
+      });
+
+      inputStream.on('error', (err: Error) => {
+        intakeStream.destroy(err);
+      });
+    } else {
+      const reader = (inputStream as ReadableStream).getReader();
+
+      const readChunk = () => {
+        reader.read().then(({ done, value }) => {
+          if (done) {
+            intakeStream.signalEnd();
+          } else {
+            intakeStream.pushData(value);
+            readChunk();
+          }
+        }).catch((err) => {
+          intakeStream.destroy(err);
+        });
+      };
+
+      readChunk();
+    }
+
+    return intakeStream;
+  }
+
+  // INSTANCE
   private signalEndBoolean = false;
   private chunkStore: T[] = [];
   public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
@@ -14,7 +53,7 @@ export class StreamIntake<T> extends plugins.stream.Readable {
   _read(size: number): void {
     // console.log('get next');
     const pushChunk = (): void => {
-      if (this.chunkStore.length > 0) {
+      while (this.chunkStore.length > 0) {
         // If push returns false, then we should stop reading
         if (!this.push(this.chunkStore.shift())) {
           return;

ts/smartstream.plugins.ts

@@ -7,7 +7,7 @@ export { stream };
 import * as lik from '@push.rocks/lik';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrx from '@push.rocks/smartrx';
-import * as webstream from '@push.rocks/webstream';
+import * as webstream from '../dist_ts_web/index.js';
 
 export { lik, smartpromise, smartrx, webstream };
 

ts_web/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @push.rocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@push.rocks/smartstream',
+  version: '3.0.45',
+  description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
+}

ts_web/classes.webduplexstream.ts

@@ -0,0 +1,156 @@
+import * as plugins from './plugins.js';
+
+
+// ========================================
+// READ
+// ========================================
+export interface IStreamToolsRead<TInput, TOutput> {
+  done: () => void;
+  write: (writeArg: TInput) => void;
+}
+
+/**
+ * the read function is called anytime
+ *   -> the WebDuplexStream is being read from
+ *      and at the same time if nothing is enqueued
+ */
+export interface IStreamReadFunction<TInput, TOutput> {
+  (toolsArg: IStreamToolsRead<TInput, TOutput>): Promise<void>;
+}
+
+// ========================================
+// WRITE
+// ========================================
+export interface IStreamToolsWrite<TInput, TOutput> {
+  truncate: () => void;
+  push: (pushArg: TOutput) => void;
+}
+
+/**
+ * the write function can return something.
+ * It is called anytime a chunk is written to the stream.
+ */
+export interface IStreamWriteFunction<TInput, TOutput> {
+  (chunkArg: TInput, toolsArg: IStreamToolsWrite<TInput, TOutput>): Promise<any>;
+}
+
+export interface IStreamFinalFunction<TInput, TOutput> {
+  (toolsArg: IStreamToolsWrite<TInput, TOutput>): Promise<TOutput>;
+}
+
+export interface WebDuplexStreamOptions<TInput, TOutput> {
+  readFunction?: IStreamReadFunction<TInput, TOutput>;
+  writeFunction?: IStreamWriteFunction<TInput, TOutput>;
+  finalFunction?: IStreamFinalFunction<TInput, TOutput>;
+}
+
+export class WebDuplexStream<TInput = any, TOutput = any> extends TransformStream<TInput, TOutput> {
+  static fromUInt8Array(uint8Array: Uint8Array): WebDuplexStream<Uint8Array, Uint8Array> {
+    const stream = new WebDuplexStream<Uint8Array, Uint8Array>({
+      writeFunction: async (chunk, { push }) => {
+        push(chunk); // Directly push the chunk as is
+        return null;
+      }
+    });
+
+    const writer = stream.writable.getWriter();
+    writer.write(uint8Array).then(() => writer.close());
+
+    return stream;
+  }
+  
+  // INSTANCE
+  options: WebDuplexStreamOptions<TInput, TOutput>;
+
+  constructor(optionsArg: WebDuplexStreamOptions<TInput, TOutput>) {
+    super({
+      async transform(chunk, controller) {
+        // Transformation logic remains unchanged
+        if (optionsArg?.writeFunction) {
+          const tools: IStreamToolsWrite<TInput, TOutput> = {
+            truncate: () => controller.terminate(),
+            push: (pushArg: TOutput) => controller.enqueue(pushArg),
+          };
+        
+          optionsArg.writeFunction(chunk, tools)
+            .then(writeReturnChunk => {
+              // the write return chunk is optional
+              // just in case the write function returns something other than void.
+              if (writeReturnChunk) {
+                controller.enqueue(writeReturnChunk);
+              }
+            })
+            .catch(err => controller.error(err));
+        } else {
+          controller.error(new Error('No write function provided'));
+        }
+      },
+      async flush(controller) {
+        // Flush logic remains unchanged
+        if (optionsArg?.finalFunction) {
+          const tools: IStreamToolsWrite<TInput, TOutput> = {
+            truncate: () => controller.terminate(),
+            push: (pipeObject) => controller.enqueue(pipeObject),
+          };
+      
+          optionsArg.finalFunction(tools)
+            .then(finalChunk => {
+              if (finalChunk) {
+                controller.enqueue(finalChunk);
+              }
+            })  
+            .catch(err => controller.error(err))
+            .finally(() => controller.terminate());
+        } else {
+          controller.terminate();
+        }
+      }
+    });
+
+    this.options = optionsArg;
+  }
+
+  // Method to create a custom readable stream that integrates the readFunction
+  // readFunction is executed whenever the stream is being read from and nothing is enqueued
+  getCustomReadableStream() {
+    const readableStream = this.readable;
+    const options = this.options;
+    const customReadable = new ReadableStream({
+      async pull(controller) {
+        const reader = readableStream.getReader();
+
+        // Check the current state of the original stream
+        const { value, done } = await reader.read();
+        reader.releaseLock();
+
+        if (done) {
+          // If the original stream is done, close the custom readable stream
+          controller.close();
+        } else {
+          if (value) {
+            // If there is data in the original stream, enqueue it and do not execute the readFunction
+            controller.enqueue(value);
+          } else if (options.readFunction) {
+            // If the original stream is empty, execute the readFunction and read again
+            await options.readFunction({
+              done: () => controller.close(),
+              write: (writeArg) => controller.enqueue(writeArg),
+            });
+            
+            const newReader = readableStream.getReader();
+            const { value: newValue, done: newDone } = await newReader.read();
+            newReader.releaseLock();
+
+            if (newDone) {
+              controller.close();
+            } else {
+              controller.enqueue(newValue);
+            }
+          }
+        }
+      }
+    });
+
+    return customReadable;
+  }
+}

ts_web/index.ts

@@ -0,0 +1,2 @@
+import './plugins.js';
+export * from './classes.webduplexstream.js';

ts_web/plugins.ts

@@ -0,0 +1,15 @@
+// @push.rocks scope
+import * as smartenv from '@push.rocks/smartenv';
+
+export {
+  smartenv,
+}
+
+// lets setup dependencies
+const smartenvInstance = new smartenv.Smartenv();
+
+await smartenvInstance.getSafeNodeModule<typeof import('stream/web')>('stream/web', async (moduleArg) => {
+  globalThis.ReadableStream = moduleArg.ReadableStream;
+  globalThis.WritableStream = moduleArg.WritableStream;
+  globalThis.TransformStream = moduleArg.TransformStream;
+})