3.0.33

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "2.0.5",
+  "version": "3.0.33",
   "private": false,
   "description": "simplifies access to node streams",
   "main": "dist_ts/index.js",
@@ -8,32 +8,31 @@
   "type": "module",
   "scripts": {
     "test": "(tstest test/)",
-    "build": "(tsbuild)"
+    "build": "(tsbuild)",
+    "buildDocs": "tsdoc"
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@gitlab.com/pushrocks/smartstream.git"
+    "url": "git+https://gitlab.com/push.rocks/smartstream.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
-    "url": "https://gitlab.com/pushrocks/smartstream/issues"
+    "url": "https://gitlab.com/push.rocks/smartstream/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/smartstream#README",
+  "homepage": "https://gitlab.com/push.rocks/smartstream#readme",
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.66",
+    "@git.zone/tsbuild": "^2.1.72",
     "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.77",
+    "@git.zone/tstest": "^1.0.86",
-    "@push.rocks/smartfile": "^10.0.33",
+    "@push.rocks/smartfile": "^11.0.4",
-    "@push.rocks/tapbundle": "^5.0.15"
+    "@push.rocks/tapbundle": "^5.0.15",
+    "@types/node": "^20.11.20"
   },
   "dependencies": {
+    "@push.rocks/lik": "^6.0.14",
     "@push.rocks/smartpromise": "^4.0.3",
-    "@push.rocks/smartrx": "^3.0.7",
+    "@push.rocks/smartrx": "^3.0.7"
-    "@types/from2": "^2.3.4",
-    "@types/through2": "^2.0.40",
-    "from2": "^2.3.0",
-    "through2": "^4.0.2"
   },
   "browserslist": [
     "last 1 chrome versions"

test/assets/writabletext.txt

@@ -48,4 +48,3 @@ hi+wow
 hi+wow
 hi+wow
 hi+wow
-noice

test/test.backpressure.ts

@@ -0,0 +1,68 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+import { SmartDuplex, type ISmartDuplexOptions, StreamWrapper } from '../ts/index.js';
+
+tap.test('should run backpressure test', async (toolsArg) => {
+  const done = toolsArg.defer();
+  async function testBackpressure() {
+    const stream1 = new SmartDuplex({
+      name: 'stream1',
+      objectMode: true,
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 10)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 1`);
+        return chunk; // Fast processing
+      },
+    });
+    const stream2 = new SmartDuplex({
+      name: 'stream2',
+      objectMode: true,
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 20)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 2`);
+        await tools.push(chunk);
+        // return chunk, optionally return ;
+      },
+    }); // This stream processes data more slowly
+    const stream3 = new SmartDuplex({
+      objectMode: true,
+      name: 'stream3',
+      writeFunction: async (chunk, tools) => {
+        await new Promise((resolve) => setTimeout(resolve, 200)); // Slow processing
+        console.log(`processed chunk ${chunk} in stream 3`);
+      },
+    });
+
+    stream1.pipe(stream2).pipe(stream3);
+
+    let backpressured = false;
+    for (let i = 0; i < 200; i++) {
+      const canContinue = stream1.write(`Chunk ${i}`, 'utf8');
+      if (!canContinue) {
+        backpressured = true;
+        console.log(`Backpressure at chunk ${i}`);
+      }
+    }
+
+    stream1.end();
+
+    stream1.on('finish', () => {
+      console.log('Stream 1 finished processing.');
+    });
+    stream2.on('finish', () => {
+      console.log('Stream 2 finished processing.');
+    });
+    stream3.on('finish', () => {
+      console.log('Stream 3 finished processing.');
+      if (!backpressured) {
+        throw new Error('No backpressure was observed.');
+      } else {
+        done.resolve();
+      }
+    });
+  }
+
+  testBackpressure();
+  await done.promise;
+});
+
+await tap.start();

test/test.smartstream.ts

@@ -1,11 +1,11 @@
 import { expect, tap } from '@push.rocks/tapbundle';
-import { SmartStream } from '../ts/smartstream.classes.smartstream.js'; // Adjust the import to your file structure
+import { SmartDuplex } from '../ts/smartstream.classes.smartduplex.js'; // Adjust the import to your file structure
 import * as smartrx from '@push.rocks/smartrx';
 import * as fs from 'fs';
 
 tap.test('should create a SmartStream from a Buffer', async () => {
   const bufferData = Buffer.from('This is a test buffer');
-  const smartStream = SmartStream.fromBuffer(bufferData);
+  const smartStream = SmartDuplex.fromBuffer(bufferData, {});
   
   let receivedData = Buffer.alloc(0);
   
@@ -21,24 +21,4 @@ tap.test('should create a SmartStream from a Buffer', async () => {
   });
 });
 
-tap.test('should create a SmartStream from an Observable', async () => {
-  const observableData = 'Observable test data';
-  const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
-  
-  const smartStream = SmartStream.fromObservable(testObservable);
-  
-  let receivedData = Buffer.alloc(0);
-
-  return new Promise<void>((resolve) => {
-    smartStream.on('data', (chunk: Buffer) => {
-      receivedData = Buffer.concat([receivedData, chunk]);
-    });
-
-    smartStream.on('end', () => {
-      expect(receivedData.toString()).toEqual(observableData);
-      resolve();
-    });
-  });
-});
-
 tap.start();

test/test.streamfunction.ts

@@ -7,45 +7,42 @@ let testIntake: smartstream.StreamIntake<string>;
 
 tap.test('should handle a read stream', async (tools) => {
   const counter = 0;
-  const testSmartstream = new smartstream.StreamWrapper([
+  const streamWrapper = new smartstream.StreamWrapper([
     smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
-    smartstream.createDuplexStream<Buffer, Buffer>(
+    new smartstream.SmartDuplex({
-      async (chunkStringArg: Buffer, streamTools) => {
+      writeFunction: async (chunkStringArg: Buffer, streamTools) => {
         // do something with the stream here
         const result = chunkStringArg.toString().substr(0, 100);
-        streamTools.pipeMore('wow =========== \n');
+        streamTools.push('wow =========== \n');
         return Buffer.from(result);
       },
-      async (tools) => {
+      finalFunction: async (tools) => {
-        // tools.pipeMore('hey, this is the end')
         return Buffer.from('this is the end');
       },
-      { objectMode: false }
-    ),
-    smartstream.createDuplexStream<Buffer, string>(async (chunkStringArg) => {
-      console.log(chunkStringArg.toString());
-      return null;
     }),
-    smartstream.cleanPipe(),
+    new smartstream.SmartDuplex({
+      writeFunction: async (chunkStringArg) => {
+        console.log(chunkStringArg.toString());
+      },
+      finalFunction: async (tools) => {
+        tools.push(null);
+      },
+    })
   ]);
-  await testSmartstream.run();
+  await streamWrapper.run();
 });
 
 tap.test('should create a valid Intake', async (tools) => {
   testIntake = new smartstream.StreamIntake<string>();
-  testIntake
+  testIntake.pipe(
-    .getReadable()
+      new smartstream.SmartDuplex({
-    .pipe(
+        objectMode: true,
-      smartstream.createDuplexStream<string, string>(
+        writeFunction: async (chunkStringArg: string, streamTools) => {
-        async (chunkString) => {
           await tools.delayFor(100);
-          console.log(chunkString);
+          console.log(chunkStringArg);
-          return chunkString;
+          return chunkStringArg;
-        },
-        async () => {
-          return 'noice';
         }
-      )
+      })
     )
     .pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
   const testFinished = tools.defer();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '2.0.5',
+  version: '3.0.33',
   description: 'simplifies access to node streams'
 }

ts/index.ts

@@ -1,4 +1,5 @@
-export * from './smartstream.classes.smartstream.js';
+export * from './smartstream.classes.smartduplex.js';
 export * from './smartstream.classes.streamwrapper.js';
 export * from './smartstream.classes.streamintake.js';
-export * from './smartstream.duplex.js';
+
+export * from './smartstream.functions.js'

ts/smartstream.classes.smartduplex.ts

@@ -0,0 +1,159 @@
+import * as plugins from './smartstream.plugins.js';
+import { Duplex, type DuplexOptions } from 'stream';
+
+export interface IStreamTools {
+  truncate: () => void;
+  push: (pipeObject: any) => Promise<void>;
+}
+
+export interface IStreamWriteFunction<T, rT> {
+  (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
+}
+
+export interface IStreamFinalFunction<rT> {
+  (toolsArg: IStreamTools): Promise<rT>;
+}
+
+export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
+  /**
+   * wether to print debug logs
+   */
+  debug?: boolean;
+  /**
+   * the name of the stream
+   */
+  name?: string;
+  /**
+   * a function that is being called to read more stuff from whereever to be processed by the stream
+   * @returns
+   */
+  readFunction?: () => Promise<void>;
+
+  /**
+   * the write function is called for every chunk that is being written to the stream
+   * it can push or return chunks (but does not have to) to be written to the readable side of the stream
+   */
+  writeFunction?: IStreamWriteFunction<TInput, TOutput>;
+  
+  /**
+   * a final function that is run at the end of the stream 
+   */
+  finalFunction?: IStreamFinalFunction<TOutput>;
+}
+
+export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
+  // STATIC
+  static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex {
+    const smartDuplex = new SmartDuplex(options);
+    process.nextTick(() => {
+      smartDuplex.push(buffer);
+      smartDuplex.push(null); // Signal the end of the data
+    });
+    return smartDuplex;
+  }
+
+  // INSTANCE
+  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
+  public options: ISmartDuplexOptions<TInput, TOutput>;
+  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
+  private debugLog(messageArg: string) {
+    if (this.options.debug) {
+      console.log(messageArg);
+    }
+  }
+
+  constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
+    super(Object.assign({
+      highWaterMark: 1,
+    }, optionsArg));
+    this.options = optionsArg;
+    this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(this.options.highWaterMark || 1)
+  }
+
+  public async _read(size: number): Promise<void> {
+    this.debugLog(`${this.options.name}: read was called`);
+    await this.backpressuredArray.waitForItems();
+    this.debugLog(`${this.options.name}: successfully waited for items.`);
+    if (this.options.readFunction) {
+      await this.options.readFunction();
+    }
+    let canPushMore = true; 
+    while(this.backpressuredArray.data.length > 0 && canPushMore) {
+      const nextChunk = this.backpressuredArray.shift();
+      canPushMore = this.push(nextChunk);
+    }
+  }
+
+  public async backpressuredPush (pushArg: TOutput) {
+    const canPushMore = this.backpressuredArray.push(pushArg);
+    if (!canPushMore) {
+      this.debugLog(`${this.options.name}: cannot push more`);
+      await this.backpressuredArray.waitForSpace();
+      this.debugLog(`${this.options.name}: can push more again`);
+    }
+  };
+
+  private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
+  // Ensure the _write method types the chunk as TInput and encodes TOutput
+  public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
+    if (!this.options.writeFunction) {
+      return callback(new Error('No stream function provided'));
+    }
+
+    let isTruncated = false;
+    const tools: IStreamTools = {
+      truncate: () => {
+        this.push(null);
+        isTruncated = true;
+        callback();
+      },
+      push: async (pushArg: TOutput) => {
+        await this.backpressuredPush(pushArg);
+      }
+    };
+
+    try {
+      const writeDeferred = plugins.smartpromise.defer();
+      this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
+      const modifiedChunk = await this.options.writeFunction(chunk, tools);
+      if (isTruncated) {
+        return;
+      }
+      if (modifiedChunk) {
+        await tools.push(modifiedChunk);
+      }
+      callback();
+      writeDeferred.resolve();
+      writeDeferred.promise.then(() => {
+        this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
+      });
+    } catch (err) {
+      callback(err);
+    }
+  }
+
+  public async _final(callback: (error?: Error | null) => void) {
+    await Promise.all(this.asyncWritePromiseObjectmap.getArray());
+    if (this.options.finalFunction) {
+      const tools: IStreamTools = {
+        truncate: () => callback(),
+        push: async (pipeObject) => {
+          this.backpressuredArray.push(pipeObject);
+        },
+      };
+
+      try {
+        const finalChunk = await this.options.finalFunction(tools);
+        if (finalChunk) {
+          this.backpressuredArray.push(finalChunk);
+        }
+      } catch (err) {
+        this.backpressuredArray.push(null);
+        callback(err);
+        return;
+      }
+    }
+    this.backpressuredArray.push(null);
+    callback();
+  }
+}

ts/smartstream.classes.smartstream.ts

@@ -1,55 +0,0 @@
-import * as plugins from './smartstream.plugins.js';
-import { Duplex, type DuplexOptions } from 'stream';
-
-export class SmartStream extends Duplex {
-  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
-
-  constructor(options?: DuplexOptions) {
-    super(options);
-  }
-
-  _read(size: number) {
-    // Implement if you need custom behavior, otherwise leave it empty
-  }
-
-  _write(chunk: any, encoding: string, callback: (error?: Error | null) => void) {
-    // Implement if you need custom behavior
-    callback();
-  }
-
-  static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
-    const smartStream = new SmartStream(options);
-    process.nextTick(() => {
-      smartStream.push(buffer);
-      smartStream.push(null); // Signal the end of the data
-    });
-    return smartStream;
-  }
-
-  static fromObservable(observable: plugins.smartrx.rxjs.Observable<any>, options?: DuplexOptions): SmartStream {
-    const smartStream = new SmartStream(options);
-    smartStream.observableSubscription = observable.subscribe({
-      next: (data) => {
-        if (!smartStream.push(data)) {
-          // Pause the observable if the stream buffer is full
-          smartStream.observableSubscription?.unsubscribe();
-          smartStream.once('drain', () => {
-            // Resume the observable when the stream buffer is drained
-            smartStream.observableSubscription?.unsubscribe();
-            smartStream.observableSubscription = observable.subscribe(data => {
-              smartStream.push(data);
-            });
-          });
-        }
-      },
-      error: (err) => {
-        smartStream.emit('error', err);
-      },
-      complete: () => {
-        smartStream.push(null); // Signal the end of the data
-      }
-    });
-
-    return smartStream;
-  }
-}

ts/smartstream.classes.streamintake.ts

@@ -1,56 +1,42 @@
 import * as plugins from './smartstream.plugins.js';
 
-export class StreamIntake<T> {
+export class StreamIntake<T> extends plugins.stream.Readable {
   private signalEndBoolean = false;
   private chunkStore: T[] = [];
-
   public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
-
   private pushedNextDeferred = plugins.smartpromise.defer();
 
-  private readableStream = plugins.from2.obj(async (size, next) => {
+  constructor(options?: plugins.stream.ReadableOptions) {
+    super({ ...options, objectMode: true }); // Ensure that we are in object mode.
+    this.pushNextObservable.push('please push next');
+  }
+
+  _read(size: number): void {
     // console.log('get next');
-    // execute without backpressure
+    const pushChunk = (): void => {
-    while (this.chunkStore.length > 0) {
+      if (this.chunkStore.length > 0) {
-      next(null, this.chunkStore.shift());
+        // If push returns false, then we should stop reading
+        if (!this.push(this.chunkStore.shift())) {
+          return;
         }
+      }
+
+      if (this.chunkStore.length === 0) {
         if (this.signalEndBoolean) {
-      next(null, null);
+          // If we're done, push null to signal the end of the stream
-    }
+          this.push(null);
-
+        } else {
-    // lets trigger backpressure handling
+          // Ask for more data and wait
           this.pushNextObservable.push('please push next');
-    await this.pushedNextDeferred.promise;
+          this.pushedNextDeferred.promise.then(() => {
-    this.pushedNextDeferred = plugins.smartpromise.defer();
+            this.pushedNextDeferred = plugins.smartpromise.defer(); // Reset the deferred
-
+            pushChunk(); // Try pushing the next chunk
-    // execute with backpressure
-    while (this.chunkStore.length > 0) {
-      next(null, this.chunkStore.shift());
-    }
-    if (this.signalEndBoolean) {
-      next(null, null);
-    }
           });
-
-  constructor() {
-    this.pushNextObservable.push('please push next');
         }
-
-  /**
-   * returns a new style readble stream
-   */
-  public getReadable() {
-    const readable = new plugins.stream.Readable({
-      objectMode: true,
-    });
-    return readable.wrap(this.readableStream);
       }
+    };
 
-  /**
+    pushChunk();
-   * returns an oldstyle readble stream
-   */
-  public getReadableStream() {
-    return this.readableStream;
   }
 
   public pushData(chunkData: T) {

ts/smartstream.classes.streamwrapper.ts

@@ -94,14 +94,3 @@ export class StreamWrapper {
     return done.promise;
   }
 }
-
-export let cleanPipe = () => {
-  return plugins.through2.obj(
-    (file, enc, cb) => {
-      cb();
-    },
-    (cb) => {
-      cb();
-    }
-  );
-};

ts/smartstream.duplex.ts

@@ -1,83 +0,0 @@
-import * as plugins from './smartstream.plugins.js';
-
-export interface ITruncateFunc {
-  (): void;
-}
-
-export interface IPipeMoreFunc {
-  (pipeObject: any): void;
-}
-
-export interface IStreamTools {
-  truncate: ITruncateFunc;
-  pipeMore: IPipeMoreFunc;
-}
-
-export interface IStreamFunction<T, rT> {
-  (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
-}
-
-export interface IStreamEndFunction<rT> {
-  (toolsArg: IStreamTools): Promise<rT>;
-}
-
-export interface IStreamOptions {
-  objectMode?: boolean;
-  readableObjectMode?: boolean;
-  writableObjectMode?: boolean;
-}
-
-export let createDuplexStream = <T, rT>(
-  funcArg: IStreamFunction<T, rT>,
-  endFuncArg?: IStreamEndFunction<rT>,
-  optionsArg: IStreamOptions = {
-    objectMode: false,
-    readableObjectMode: true,
-    writableObjectMode: true,
-  }
-) => {
-  return plugins.through2(
-    optionsArg,
-    function (chunk, enc, cb) {
-      let truncated = false;
-      const tools: IStreamTools = {
-        truncate: () => {
-          truncated = true;
-          cb(null, null);
-        },
-        pipeMore: (pipeObject) => {
-          this.push(pipeObject);
-        },
-      };
-      const asyncWrapper = async () => {
-        const resultChunk: rT = await funcArg(chunk, tools);
-        if (!truncated) {
-          cb(null, resultChunk);
-        }
-      };
-      asyncWrapper().catch((err) => {
-        console.log(err);
-      });
-    },
-    function (cb) {
-      const tools: IStreamTools = {
-        truncate: () => {
-          cb();
-        },
-        pipeMore: (pushArg) => {
-          this.push(pushArg);
-        },
-      };
-      const asyncWrapper = async () => {
-        if (endFuncArg) {
-          const result = await endFuncArg(tools);
-          this.push(result);
-        }
-        cb();
-      };
-      asyncWrapper().catch((err) => {
-        console.log(err);
-      });
-    }
-  );
-};

ts/smartstream.functions.ts

@@ -0,0 +1,30 @@
+import { Transform, type TransformCallback, type TransformOptions } from 'stream';
+import { SmartDuplex } from './smartstream.classes.smartduplex.js';
+
+export interface AsyncTransformFunction<TInput, TOutput> {
+  (chunkArg: TInput): Promise<TOutput>;
+}
+
+export function createTransformFunction<TInput, TOutput>(
+  asyncFunction: AsyncTransformFunction<TInput, TOutput>,
+  options?: TransformOptions
+): SmartDuplex {
+  const smartDuplexStream = new SmartDuplex({
+    ...options,
+    writeFunction: async (chunkArg, toolsArg) => {
+      const result = await asyncFunction(chunkArg);
+      return result;
+    }
+  });
+
+  return smartDuplexStream;
+}
+
+export const createPassThrough = () => {
+  return new SmartDuplex({
+    objectMode: true,
+    writeFunction: async (chunkArg, toolsArg) => {
+      return chunkArg;
+    }
+  })
+}

ts/smartstream.plugins.ts

@@ -4,13 +4,9 @@ import * as stream from 'stream';
 export { stream };
 
 // pushrocks scope
+import * as lik from '@push.rocks/lik';
 import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartrx from '@push.rocks/smartrx';
 
-export { smartpromise, smartrx };
+export { lik, smartpromise, smartrx };
 
-// thirdparty
-import from2 from 'from2';
-import through2 from 'through2';
-
-export { from2, through2 };