2.0.7

.gitlab-ci.yml

@@ -123,7 +123,7 @@ pages:
   stage: metadata
   script:
     - npmci node install lts
-    - npmci command npm install -g @gitzone/tsdoc
+    - npmci command npm install -g @git.zone/tsdoc
     - npmci npm prepare
     - npmci npm install
     - npmci command tsdoc

npmextra.json

@@ -7,10 +7,10 @@
     "projectType": "npm",
     "module": {
       "githost": "gitlab.com",
-      "gitscope": "pushrocks",
+      "gitscope": "push.rocks",
       "gitrepo": "smartstream",
       "description": "simplifies access to node streams",
-      "npmPackagename": "@pushrocks/smartstream",
+      "npmPackagename": "@push.rocks/smartstream",
       "license": "MIT"
     }
   }

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@pushrocks/smartstream",
-  "version": "2.0.1",
+  "name": "@push.rocks/smartstream",
+  "version": "2.0.7",
   "private": false,
   "description": "simplifies access to node streams",
   "main": "dist_ts/index.js",
@@ -21,15 +21,18 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartstream#README",
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.61",
-    "@gitzone/tstest": "^1.0.70",
-    "@pushrocks/tapbundle": "^5.0.3",
-    "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.18.0"
+    "@git.zone/tsbuild": "^2.1.66",
+    "@git.zone/tsrun": "^1.2.44",
+    "@git.zone/tstest": "^1.0.77",
+    "@push.rocks/smartfile": "^10.0.33",
+    "@push.rocks/tapbundle": "^5.0.15"
   },
   "dependencies": {
-    "@pushrocks/smartpromise": "^3.1.7",
-    "@types/through2": "^2.0.36",
+    "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartrx": "^3.0.7",
+    "@types/from2": "^2.3.4",
+    "@types/through2": "^2.0.40",
+    "from2": "^2.3.0",
     "through2": "^4.0.2"
   },
   "browserslist": [

test/assets/writabletext.txt

@@ -0,0 +1,51 @@
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+hi+wow
+noice

test/test.smartstream.ts

@@ -0,0 +1,44 @@
+import { expect, tap } from '@push.rocks/tapbundle';
+import { SmartStream } from '../ts/smartstream.classes.smartstream.js'; // Adjust the import to your file structure
+import * as smartrx from '@push.rocks/smartrx';
+import * as fs from 'fs';
+
+tap.test('should create a SmartStream from a Buffer', async () => {
+  const bufferData = Buffer.from('This is a test buffer');
+  const smartStream = SmartStream.fromBuffer(bufferData);
+  
+  let receivedData = Buffer.alloc(0);
+  
+  return new Promise<void>((resolve) => {
+    smartStream.on('data', (chunk: Buffer) => {
+      receivedData = Buffer.concat([receivedData, chunk]);
+    });
+
+    smartStream.on('end', () => {
+      expect(receivedData.toString()).toEqual(bufferData.toString());
+      resolve();
+    });
+  });
+});
+
+tap.test('should create a SmartStream from an Observable', async () => {
+  const observableData = 'Observable test data';
+  const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
+  
+  const smartStream = SmartStream.fromObservable(testObservable);
+  
+  let receivedData = Buffer.alloc(0);
+
+  return new Promise<void>((resolve) => {
+    smartStream.on('data', (chunk: Buffer) => {
+      receivedData = Buffer.concat([receivedData, chunk]);
+    });
+
+    smartStream.on('end', () => {
+      expect(receivedData.toString()).toEqual(observableData);
+      resolve();
+    });
+  });
+});
+
+tap.start();

test/test.streamfunction.ts

@@ -0,0 +1,68 @@
+import { expect, tap } from '@push.rocks/tapbundle';
+import * as smartfile from '@push.rocks/smartfile';
+
+import * as smartstream from '../ts/index.js';
+
+let testIntake: smartstream.StreamIntake<string>;
+
+tap.test('should handle a read stream', async (tools) => {
+  const counter = 0;
+  const testSmartstream = new smartstream.StreamWrapper([
+    smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
+    smartstream.createDuplexStream<Buffer, Buffer>(
+      async (chunkStringArg: Buffer, streamTools) => {
+        // do something with the stream here
+        const result = chunkStringArg.toString().substr(0, 100);
+        streamTools.pipeMore('wow =========== \n');
+        return Buffer.from(result);
+      },
+      async (tools) => {
+        // tools.pipeMore('hey, this is the end')
+        return Buffer.from('this is the end');
+      },
+      { objectMode: false }
+    ),
+    smartstream.createDuplexStream<Buffer, string>(async (chunkStringArg) => {
+      console.log(chunkStringArg.toString());
+      return null;
+    }),
+    smartstream.cleanPipe(),
+  ]);
+  await testSmartstream.run();
+});
+
+tap.test('should create a valid Intake', async (tools) => {
+  testIntake = new smartstream.StreamIntake<string>();
+  testIntake
+    .getReadable()
+    .pipe(
+      smartstream.createDuplexStream<string, string>(
+        async (chunkString) => {
+          await tools.delayFor(100);
+          console.log(chunkString);
+          return chunkString;
+        },
+        async () => {
+          return 'noice';
+        }
+      )
+    )
+    .pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
+  const testFinished = tools.defer();
+  let counter = 0;
+  testIntake.pushNextObservable.subscribe(() => {
+    if (counter < 50) {
+      counter++;
+      testIntake.pushData('hi');
+      testIntake.pushData('+wow');
+      testIntake.pushData('\n');
+    } else {
+      testIntake.signalEnd();
+      testFinished.resolve();
+    }
+  });
+  await testFinished.promise;
+  testIntake.signalEnd();
+});
+
+tap.start();

test/test.ts

@@ -1,13 +1,13 @@
-import fs from 'fs';
-import { expect, tap } from '@pushrocks/tapbundle';
+import * as smartfile from '@push.rocks/smartfile';
+import { expect, tap } from '@push.rocks/tapbundle';
 
-import * as smartstream from '../ts/index.js';
+import * as smartstream from '../ts/smartstream.classes.streamwrapper.js';
 
-let testSmartstream: smartstream.Smartstream;
+let testSmartstream: smartstream.StreamWrapper;
 tap.test('should combine a stream', async () => {
-  testSmartstream = new smartstream.Smartstream([
-    fs.createReadStream('./test/assets/test.md'),
-    fs.createWriteStream('./test/assets/testCopy.md'),
+  testSmartstream = new smartstream.StreamWrapper([
+    smartfile.fsStream.createReadStream('./test/assets/test.md'),
+    smartfile.fsStream.createWriteStream('./test/assets/testCopy.md'),
   ]);
   await testSmartstream.run();
 });

ts/00_commitinfo_data.ts

@@ -0,0 +1,8 @@
+/**
+ * autocreated commitinfo by @pushrocks/commitinfo
+ */
+export const commitinfo = {
+  name: '@push.rocks/smartstream',
+  version: '2.0.7',
+  description: 'simplifies access to node streams'
+}

ts/index.ts

@@ -1,107 +1,4 @@
-import * as plugins from './smartstream.plugins.js';
-
-// interfaces
-import { Transform } from 'stream';
-
-export interface IErrorFunction {
-  (err: Error): any;
-}
-
-export interface ICustomEventFunction {
-  (): any;
-}
-
-export interface ICustomEventObject {
-  eventName: string;
-  eventFunction: ICustomEventFunction;
-}
-
-/**
- * class Smartstream handles
- */
-export class Smartstream {
-  private streamArray: Array<plugins.stream.Duplex> = [];
-  private customEventObjectArray: ICustomEventObject[] = [];
-  private streamStartedDeferred = plugins.smartpromise.defer();
-
-  /**
-   * constructor
-   */
-  constructor(streamArrayArg: any[]) {
-    this.streamArray = streamArrayArg;
-  }
-
-  /**
-   * make something with the stream itself
-   */
-  streamStarted(): Promise<any> {
-    return this.streamStartedDeferred.promise;
-  }
-
-  /**
-   * attach listener to custom event
-   */
-  onCustomEvent(eventNameArg: string, eventFunctionArg: ICustomEventFunction) {
-    this.customEventObjectArray.push({
-      eventName: eventNameArg,
-      eventFunction: eventFunctionArg,
-    });
-  }
-
-  /**
-   * run the stream
-   * @returns Promise
-   */
-  run(): Promise<void> {
-    const done = plugins.smartpromise.defer<void>();
-
-    // clone Array
-    const streamExecutionArray: Array<plugins.stream.Duplex> = [];
-    for (const streamItem of this.streamArray) {
-      streamExecutionArray.push(streamItem);
-    }
-
-    // combine the stream
-    let finalStream = null;
-    let firstIteration: boolean = true;
-    for (const stream of streamExecutionArray) {
-      if (firstIteration === true) {
-        finalStream = stream;
-      }
-      stream.on('error', (err) => {
-        done.reject(err);
-      });
-      for (const customEventObject of this.customEventObjectArray) {
-        stream.on(customEventObject.eventName, customEventObject.eventFunction);
-      }
-      if (!firstIteration) {
-        finalStream = finalStream.pipe(stream);
-      }
-      firstIteration = false;
-    }
-
-    this.streamStartedDeferred.resolve();
-
-    finalStream.on('end', () => {
-      done.resolve();
-    });
-    finalStream.on('close', () => {
-      done.resolve();
-    });
-    finalStream.on('finish', () => {
-      done.resolve();
-    });
-    return done.promise;
-  }
-}
-
-export let cleanPipe = () => {
-  return plugins.through2.obj(
-    (file, enc, cb) => {
-      cb();
-    },
-    (cb) => {
-      cb();
-    }
-  );
-};
+export * from './smartstream.classes.smartstream.js';
+export * from './smartstream.classes.streamwrapper.js';
+export * from './smartstream.classes.streamintake.js';
+export * from './smartstream.duplex.js';

ts/smartstream.classes.smartstream.ts

@@ -0,0 +1,55 @@
+import * as plugins from './smartstream.plugins.js';
+import { Duplex, type DuplexOptions } from 'stream';
+
+export class SmartStream extends Duplex {
+  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
+
+  constructor(options?: DuplexOptions) {
+    super(options);
+  }
+
+  _read(size: number) {
+    // Implement if you need custom behavior, otherwise leave it empty
+  }
+
+  _write(chunk: any, encoding: string, callback: (error?: Error | null) => void) {
+    // Implement if you need custom behavior
+    callback();
+  }
+
+  static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
+    const smartStream = new SmartStream(options);
+    process.nextTick(() => {
+      smartStream.push(buffer);
+      smartStream.push(null); // Signal the end of the data
+    });
+    return smartStream;
+  }
+
+  static fromObservable(observable: plugins.smartrx.rxjs.Observable<any>, options?: DuplexOptions): SmartStream {
+    const smartStream = new SmartStream(options);
+    smartStream.observableSubscription = observable.subscribe({
+      next: (data) => {
+        if (!smartStream.push(data)) {
+          // Pause the observable if the stream buffer is full
+          smartStream.observableSubscription?.unsubscribe();
+          smartStream.once('drain', () => {
+            // Resume the observable when the stream buffer is drained
+            smartStream.observableSubscription?.unsubscribe();
+            smartStream.observableSubscription = observable.subscribe(data => {
+              smartStream.push(data);
+            });
+          });
+        }
+      },
+      error: (err) => {
+        smartStream.emit('error', err);
+      },
+      complete: () => {
+        smartStream.push(null); // Signal the end of the data
+      }
+    });
+
+    return smartStream;
+  }
+}

ts/smartstream.classes.streamintake.ts

@@ -0,0 +1,66 @@
+import * as plugins from './smartstream.plugins.js';
+
+export class StreamIntake<T> {
+  private signalEndBoolean = false;
+  private chunkStore: T[] = [];
+
+  public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
+
+  private pushedNextDeferred = plugins.smartpromise.defer();
+
+  private readableStream = plugins.from2.obj(async (size, next) => {
+    // console.log('get next');
+    // execute without backpressure
+    while (this.chunkStore.length > 0) {
+      next(null, this.chunkStore.shift());
+    }
+    if (this.signalEndBoolean) {
+      next(null, null);
+    }
+
+    // lets trigger backpressure handling
+    this.pushNextObservable.push('please push next');
+    await this.pushedNextDeferred.promise;
+    this.pushedNextDeferred = plugins.smartpromise.defer();
+
+    // execute with backpressure
+    while (this.chunkStore.length > 0) {
+      next(null, this.chunkStore.shift());
+    }
+    if (this.signalEndBoolean) {
+      next(null, null);
+    }
+  });
+
+  constructor() {
+    this.pushNextObservable.push('please push next');
+  }
+
+  /**
+   * returns a new style readble stream
+   */
+  public getReadable() {
+    const readable = new plugins.stream.Readable({
+      objectMode: true,
+    });
+    return readable.wrap(this.readableStream);
+  }
+
+  /**
+   * returns an oldstyle readble stream
+   */
+  public getReadableStream() {
+    return this.readableStream;
+  }
+
+  public pushData(chunkData: T) {
+    this.chunkStore.push(chunkData);
+    this.pushedNextDeferred.resolve();
+  }
+
+  public signalEnd() {
+    this.signalEndBoolean = true;
+    this.pushedNextDeferred.resolve();
+    this.pushNextObservable.signalComplete();
+  }
+}

ts/smartstream.classes.streamwrapper.ts

@@ -0,0 +1,107 @@
+import * as plugins from './smartstream.plugins.js';
+
+// interfaces
+import { Transform } from 'stream';
+
+export interface IErrorFunction {
+  (err: Error): any;
+}
+
+export interface ICustomEventFunction {
+  (): any;
+}
+
+export interface ICustomEventObject {
+  eventName: string;
+  eventFunction: ICustomEventFunction;
+}
+
+/**
+ * class Smartstream handles
+ */
+export class StreamWrapper {
+  private streamArray: Array<plugins.stream.Duplex> = [];
+  private customEventObjectArray: ICustomEventObject[] = [];
+  private streamStartedDeferred = plugins.smartpromise.defer();
+
+  /**
+   * constructor
+   */
+  constructor(streamArrayArg: any[]) {
+    this.streamArray = streamArrayArg;
+  }
+
+  /**
+   * make something with the stream itself
+   */
+  streamStarted(): Promise<any> {
+    return this.streamStartedDeferred.promise;
+  }
+
+  /**
+   * attach listener to custom event
+   */
+  onCustomEvent(eventNameArg: string, eventFunctionArg: ICustomEventFunction) {
+    this.customEventObjectArray.push({
+      eventName: eventNameArg,
+      eventFunction: eventFunctionArg,
+    });
+  }
+
+  /**
+   * run the stream
+   * @returns Promise
+   */
+  run(): Promise<void> {
+    const done = plugins.smartpromise.defer<void>();
+
+    // clone Array
+    const streamExecutionArray: Array<plugins.stream.Duplex> = [];
+    for (const streamItem of this.streamArray) {
+      streamExecutionArray.push(streamItem);
+    }
+
+    // combine the stream
+    let finalStream = null;
+    let firstIteration: boolean = true;
+    for (const stream of streamExecutionArray) {
+      if (firstIteration === true) {
+        finalStream = stream;
+      }
+      stream.on('error', (err) => {
+        done.reject(err);
+      });
+      for (const customEventObject of this.customEventObjectArray) {
+        stream.on(customEventObject.eventName, customEventObject.eventFunction);
+      }
+      if (!firstIteration) {
+        finalStream = finalStream.pipe(stream);
+      }
+      firstIteration = false;
+    }
+
+    this.streamStartedDeferred.resolve();
+
+    finalStream.on('end', () => {
+      done.resolve();
+    });
+    finalStream.on('close', () => {
+      done.resolve();
+    });
+    finalStream.on('finish', () => {
+      done.resolve();
+    });
+    return done.promise;
+  }
+}
+
+export let cleanPipe = () => {
+  return plugins.through2.obj(
+    (file, enc, cb) => {
+      cb();
+    },
+    (cb) => {
+      cb();
+    }
+  );
+};

ts/smartstream.duplex.ts

@@ -0,0 +1,83 @@
+import * as plugins from './smartstream.plugins.js';
+
+export interface ITruncateFunc {
+  (): void;
+}
+
+export interface IPipeMoreFunc {
+  (pipeObject: any): void;
+}
+
+export interface IStreamTools {
+  truncate: ITruncateFunc;
+  pipeMore: IPipeMoreFunc;
+}
+
+export interface IStreamFunction<T, rT> {
+  (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
+}
+
+export interface IStreamEndFunction<rT> {
+  (toolsArg: IStreamTools): Promise<rT>;
+}
+
+export interface IStreamOptions {
+  objectMode?: boolean;
+  readableObjectMode?: boolean;
+  writableObjectMode?: boolean;
+}
+
+export let createDuplexStream = <T, rT>(
+  funcArg: IStreamFunction<T, rT>,
+  endFuncArg?: IStreamEndFunction<rT>,
+  optionsArg: IStreamOptions = {
+    objectMode: false,
+    readableObjectMode: true,
+    writableObjectMode: true,
+  }
+) => {
+  return plugins.through2(
+    optionsArg,
+    function (chunk, enc, cb) {
+      let truncated = false;
+      const tools: IStreamTools = {
+        truncate: () => {
+          truncated = true;
+          cb(null, null);
+        },
+        pipeMore: (pipeObject) => {
+          this.push(pipeObject);
+        },
+      };
+      const asyncWrapper = async () => {
+        const resultChunk: rT = await funcArg(chunk, tools);
+        if (!truncated) {
+          cb(null, resultChunk);
+        }
+      };
+      asyncWrapper().catch((err) => {
+        console.log(err);
+      });
+    },
+    function (cb) {
+      const tools: IStreamTools = {
+        truncate: () => {
+          cb();
+        },
+        pipeMore: (pushArg) => {
+          this.push(pushArg);
+        },
+      };
+      const asyncWrapper = async () => {
+        if (endFuncArg) {
+          const result = await endFuncArg(tools);
+          this.push(result);
+        }
+        cb();
+      };
+      asyncWrapper().catch((err) => {
+        console.log(err);
+      });
+    }
+  );
+};

ts/smartstream.plugins.ts

@@ -1,12 +1,16 @@
+// node native
 import * as stream from 'stream';
 
 export { stream };
 
 // pushrocks scope
-import * as smartpromise from '@pushrocks/smartpromise';
-export { smartpromise };
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
+
+export { smartpromise, smartrx };
 
 // thirdparty
-import * as through2 from 'through2';
+import from2 from 'from2';
+import through2 from 'through2';
 
-export { through2 };
+export { from2, through2 };

tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "useDefineForClassFields": false,
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
+}