3.0.17

.gitea/workflows/default_nottags.yaml

@@ -0,0 +1,66 @@
+name: Default (not tags)
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install pnpm and npmci
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+
+      - name: Run npm prepare
+        run: npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build

.gitea/workflows/default_tags.yaml

@@ -0,0 +1,124 @@
+name: Default (tags)
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
+  NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
+  NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
+  NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
+  NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
+  NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Audit production dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --prod
+        continue-on-error: true
+
+      - name: Audit development dependencies
+        run: |
+          npmci command npm config set registry https://registry.npmjs.org
+          npmci command pnpm audit --audit-level=high --dev
+        continue-on-error: true
+
+  test:
+    if: ${{ always() }}
+    needs: security
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Test stable
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm test
+
+      - name: Test build
+        run: |
+          npmci node install stable
+          npmci npm install
+          npmci npm build
+
+  release:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Release
+        run: |
+          npmci node install stable
+          npmci npm publish
+
+  metadata:
+    needs: test
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ env.IMAGE }}
+    continue-on-error: true
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Prepare
+        run: |
+          pnpm install -g pnpm
+          pnpm install -g @shipzone/npmci
+          npmci npm prepare
+
+      - name: Code quality
+        run: |
+          npmci command npm install -g typescript
+          npmci npm install
+
+      - name: Trigger
+        run: npmci trigger
+
+      - name: Build docs and upload artifacts
+        run: |
+          npmci node install stable
+          npmci npm install
+          pnpm install -g @git.zone/tsdoc
+          npmci command tsdoc
+        continue-on-error: true

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartstream",
-  "version": "3.0.1",
+  "version": "3.0.17",
   "private": false,
   "description": "simplifies access to node streams",
   "main": "dist_ts/index.js",
@@ -8,32 +8,30 @@
   "type": "module",
   "scripts": {
     "test": "(tstest test/)",
-    "build": "(tsbuild)"
+    "build": "(tsbuild)",
+    "buildDocs": "tsdoc"
   },
   "repository": {
     "type": "git",
-    "url": "git+ssh://git@gitlab.com/pushrocks/smartstream.git"
+    "url": "git+https://gitlab.com/push.rocks/smartstream.git"
   },
   "author": "Lossless GmbH",
   "license": "MIT",
   "bugs": {
-    "url": "https://gitlab.com/pushrocks/smartstream/issues"
+    "url": "https://gitlab.com/push.rocks/smartstream/issues"
   },
-  "homepage": "https://gitlab.com/pushrocks/smartstream#README",
+  "homepage": "https://gitlab.com/push.rocks/smartstream#readme",
   "devDependencies": {
     "@git.zone/tsbuild": "^2.1.66",
     "@git.zone/tsrun": "^1.2.44",
-    "@git.zone/tstest": "^1.0.77",
+    "@git.zone/tstest": "^1.0.84",
-    "@push.rocks/smartfile": "^10.0.33",
+    "@push.rocks/smartfile": "^11.0.0",
-    "@push.rocks/tapbundle": "^5.0.15"
+    "@push.rocks/tapbundle": "^5.0.15",
+    "@types/node": "^20.9.0"
   },
   "dependencies": {
     "@push.rocks/smartpromise": "^4.0.3",
-    "@push.rocks/smartrx": "^3.0.7",
+    "@push.rocks/smartrx": "^3.0.7"
-    "@types/from2": "^2.3.4",
-    "@types/through2": "^2.0.40",
-    "from2": "^2.3.0",
-    "through2": "^4.0.2"
   },
   "browserslist": [
     "last 1 chrome versions"

test/test.smartstream.ts

@@ -5,7 +5,9 @@ import * as fs from 'fs';
 
 tap.test('should create a SmartStream from a Buffer', async () => {
   const bufferData = Buffer.from('This is a test buffer');
-  const smartStream = SmartDuplex.fromBuffer(bufferData);
+  const smartStream = SmartDuplex.fromBuffer(bufferData, {
+    handleBackpressure: false,
+  });
   
   let receivedData = Buffer.alloc(0);
   
@@ -25,7 +27,9 @@ tap.test('should create a SmartStream from an Observable', async () => {
   const observableData = 'Observable test data';
   const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
   
-  const smartStream = SmartDuplex.fromObservable(testObservable);
+  const smartStream = SmartDuplex.fromObservable(testObservable, {
+    handleBackpressure: false,
+  });
   
   let receivedData = Buffer.alloc(0);
 

test/test.streamfunction.ts

@@ -10,25 +10,24 @@ tap.test('should handle a read stream', async (tools) => {
   const streamWrapper = new smartstream.StreamWrapper([
     smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
     new smartstream.SmartDuplex({
-      writeAndTransformFunction: async (chunkStringArg: Buffer, streamTools) => {
+      writeFunction: async (chunkStringArg: Buffer, streamTools) => {
         // do something with the stream here
         const result = chunkStringArg.toString().substr(0, 100);
         streamTools.push('wow =========== \n');
         return Buffer.from(result);
       },
-      streamEndFunction: async (tools) => {
+      finalFunction: async (tools) => {
         return Buffer.from('this is the end');
       },
     }),
     new smartstream.SmartDuplex({
-      writeAndTransformFunction: async (chunkStringArg) => {
+      writeFunction: async (chunkStringArg) => {
         console.log(chunkStringArg.toString());
       },
-      streamEndFunction: async (tools) => {
+      finalFunction: async (tools) => {
         tools.push(null);
       },
-    }),
+    })
-    smartstream.cleanPipe(),
   ]);
   // await streamWrapper.run();
 });
@@ -38,7 +37,7 @@ tap.test('should create a valid Intake', async (tools) => {
   testIntake.pipe(
       new smartstream.SmartDuplex({
         objectMode: true,
-        writeAndTransformFunction: async (chunkStringArg: string, streamTools) => {
+        writeFunction: async (chunkStringArg: string, streamTools) => {
           await tools.delayFor(100);
           console.log(chunkStringArg);
           return chunkStringArg;

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.0.1',
+  version: '3.0.17',
   description: 'simplifies access to node streams'
 }

ts/index.ts

@@ -2,3 +2,5 @@ export * from './smartstream.classes.passthrough.js';
 export * from './smartstream.classes.smartduplex.js';
 export * from './smartstream.classes.streamwrapper.js';
 export * from './smartstream.classes.streamintake.js';
+
+export * from './smartstream.functions.js'

ts/smartstream.classes.passthrough.ts

@@ -13,7 +13,9 @@ export class PassThrough extends plugins.stream.Duplex {
     if (this.push(chunk, encoding)) {
       callback();
     } else {
-      this.once('drain', callback);
+      this.once('drain', () => {
+        callback();
+      });
     }
   }
 }

ts/smartstream.classes.smartduplex.ts

@@ -6,40 +6,41 @@ export interface IStreamTools {
   push: (pipeObject: any) => void;
 }
 
-export interface IWriteAndTransformFunction<T, rT> {
+export interface IStreamWriteFunction<T, rT> {
   (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
 }
 
-export interface IStreamEndFunction<rT> {
+export interface IStreamFinalFunction<rT> {
   (toolsArg: IStreamTools): Promise<rT>;
 }
 
-export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
+export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
+  handleBackpressure?: boolean;
   readFunction?: () => Promise<void>;
-  writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
+  writeFunction?: IStreamWriteFunction<TInput, TOutput>;
-  streamEndFunction?: IStreamEndFunction<TOutput>;
+  finalFunction?: IStreamFinalFunction<TOutput>;
   // Add other custom options if necessary
 }
 
 export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   // STATIC
-  static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartDuplex {
+  static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex {
-    const smartStream = new SmartDuplex(options);
+    const smartDuplex = new SmartDuplex(options);
     process.nextTick(() => {
-      smartStream.push(buffer);
+      smartDuplex.push(buffer);
-      smartStream.push(null); // Signal the end of the data
+      smartDuplex.push(null); // Signal the end of the data
     });
-    return smartStream;
+    return smartDuplex;
   }
 
   static fromObservable(
     observable: plugins.smartrx.rxjs.Observable<any>,
-    options?: DuplexOptions
+    options?: ISmartDuplexOptions<any, any>
   ): SmartDuplex {
     const smartStream = new SmartDuplex(options);
     smartStream.observableSubscription = observable.subscribe({
       next: (data) => {
-        if (!smartStream.push(data)) {
+        if (!smartStream.push(data) && smartStream.handleBackpressure) {
           // Pause the observable if the stream buffer is full
           smartStream.observableSubscription?.unsubscribe();
           smartStream.once('drain', () => {
@@ -109,15 +110,17 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
 
   // INSTANCE
   private readFunction?: () => Promise<void>;
-  private writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
+  private handleBackpressure: boolean;
-  private streamEndFunction?: IStreamEndFunction<TOutput>;
+  private writeFunction?: IStreamWriteFunction<TInput, TOutput>;
+  private finalFunction?: IStreamFinalFunction<TOutput>;
   private observableSubscription?: plugins.smartrx.rxjs.Subscription;
 
-  constructor(optionsArg?: SmartStreamOptions<TInput, TOutput>) {
+  constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
     super(optionsArg);
     this.readFunction = optionsArg?.readFunction;
-    this.writeAndTransformFunction = optionsArg?.writeAndTransformFunction;
+    this.writeFunction = optionsArg?.writeFunction;
-    this.streamEndFunction = optionsArg?.streamEndFunction;
+    this.finalFunction = optionsArg?.finalFunction;
+    this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
   }
 
   public async _read(size: number): Promise<void> {
@@ -126,9 +129,24 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     }
   }
 
+  public notBackpressured = true;
+  public push(chunkArg?: TOutput | null): boolean {
+    const result = super.push(chunkArg);
+    if (!result && this.handleBackpressure) {
+      this.notBackpressured = false;
+      this.pause();
+      // Listen for 'drain' event to resume
+      this.once('drain', () => {
+        this.notBackpressured = true;
+        this.resume(); // Resume the source of data
+      });
+    }
+    return result;
+  }
+
   // Ensure the _write method types the chunk as TInput and encodes TOutput
   public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
-    if (!this.writeAndTransformFunction) {
+    if (!this.writeFunction) {
       return callback(new Error('No stream function provided'));
     }
 
@@ -141,37 +159,36 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     };
 
     try {
-      const modifiedChunk = await this.writeAndTransformFunction(chunk, tools);
+      const modifiedChunk = await this.writeFunction(chunk, tools);
       if (modifiedChunk) {
-        if (!this.push(modifiedChunk)) {
+        this.push(modifiedChunk);
-          // Handle backpressure if necessary
-        }
       }
       callback();
     } catch (err) {
       callback(err);
     }
+    return this.notBackpressured;
   }
 
   public async _final(callback: (error?: Error | null) => void) {
-    if (this.streamEndFunction) {
+    if (this.finalFunction) {
       const tools: IStreamTools = {
         truncate: () => callback(),
         push: (pipeObject) => this.push(pipeObject),
       };
 
       try {
-        const finalChunk = await this.streamEndFunction(tools);
+        const finalChunk = await this.finalFunction(tools);
         if (finalChunk) {
           this.push(finalChunk);
         }
-        callback();
       } catch (err) {
         callback(err);
       }
     } else {
-      this.push(null),
+      // nothing here
+    }
+    this.push(null);
     callback();
   }
 }
-}

ts/smartstream.classes.streamwrapper.ts

@@ -94,14 +94,3 @@ export class StreamWrapper {
     return done.promise;
   }
 }
-
-export let cleanPipe = () => {
-  return plugins.through2.obj(
-    (file, enc, cb) => {
-      cb();
-    },
-    (cb) => {
-      cb();
-    }
-  );
-};

ts/smartstream.functions.ts

@@ -0,0 +1,21 @@
+import { Transform, type TransformCallback, type TransformOptions } from 'stream';
+import { SmartDuplex } from './smartstream.classes.smartduplex.js';
+
+export interface AsyncTransformFunction<TInput, TOutput> {
+  (chunkArg: TInput): Promise<TOutput>;
+}
+
+export function createTransformFunction<TInput, TOutput>(
+  asyncFunction: AsyncTransformFunction<TInput, TOutput>,
+  options?: TransformOptions
+): SmartDuplex {
+  const smartDuplexStream = new SmartDuplex({
+    ...options,
+    writeFunction: async (chunkArg, toolsArg) => {
+      const result = await asyncFunction(chunkArg);
+      return result;
+    }
+  });
+
+  return smartDuplexStream;
+}

ts/smartstream.plugins.ts

@@ -9,8 +9,3 @@ import * as smartrx from '@push.rocks/smartrx';
 
 export { smartpromise, smartrx };
 
-// thirdparty
-import from2 from 'from2';
-import through2 from 'through2';
-
-export { from2, through2 };