3.0.0

.gitlab-ci.yml

@@ -123,7 +123,7 @@ pages:
   stage: metadata
   script:
     - npmci node install lts
-    - npmci command npm install -g @gitzone/tsdoc
+    - npmci command npm install -g @git.zone/tsdoc
     - npmci npm prepare
     - npmci npm install
     - npmci command tsdoc

npmextra.json

@@ -7,10 +7,10 @@
     "projectType": "npm",
     "module": {
       "githost": "gitlab.com",
-      "gitscope": "pushrocks",
+      "gitscope": "push.rocks",
       "gitrepo": "smartstream",
       "description": "simplifies access to node streams",
-      "npmPackagename": "@pushrocks/smartstream",
+      "npmPackagename": "@push.rocks/smartstream",
       "license": "MIT"
     }
   }

package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@pushrocks/smartstream",
-  "version": "2.0.2",
+  "name": "@push.rocks/smartstream",
+  "version": "3.0.0",
   "private": false,
   "description": "simplifies access to node streams",
   "main": "dist_ts/index.js",
@@ -21,18 +21,17 @@
   },
   "homepage": "https://gitlab.com/pushrocks/smartstream#README",
   "devDependencies": {
-    "@gitzone/tsbuild": "^2.1.63",
-    "@gitzone/tstest": "^1.0.71",
-    "@pushrocks/smartfile": "^10.0.0",
-    "@pushrocks/tapbundle": "^5.0.3",
-    "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.18.0"
+    "@git.zone/tsbuild": "^2.1.66",
+    "@git.zone/tsrun": "^1.2.44",
+    "@git.zone/tstest": "^1.0.77",
+    "@push.rocks/smartfile": "^10.0.33",
+    "@push.rocks/tapbundle": "^5.0.15"
   },
   "dependencies": {
-    "@pushrocks/smartpromise": "^3.1.7",
-    "@pushrocks/smartrx": "^2.0.25",
-    "@types/from2": "^2.3.1",
-    "@types/through2": "^2.0.36",
+    "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartrx": "^3.0.7",
+    "@types/from2": "^2.3.4",
+    "@types/through2": "^2.0.40",
     "from2": "^2.3.0",
     "through2": "^4.0.2"
   },

test/test.smartstream.ts

@@ -0,0 +1,44 @@
+import { expect, tap } from '@push.rocks/tapbundle';
+import { SmartStream } from '../ts/smartstream.classes.smartstream.js'; // Adjust the import to your file structure
+import * as smartrx from '@push.rocks/smartrx';
+import * as fs from 'fs';
+
+tap.test('should create a SmartStream from a Buffer', async () => {
+  const bufferData = Buffer.from('This is a test buffer');
+  const smartStream = SmartStream.fromBuffer(bufferData);
+  
+  let receivedData = Buffer.alloc(0);
+  
+  return new Promise<void>((resolve) => {
+    smartStream.on('data', (chunk: Buffer) => {
+      receivedData = Buffer.concat([receivedData, chunk]);
+    });
+
+    smartStream.on('end', () => {
+      expect(receivedData.toString()).toEqual(bufferData.toString());
+      resolve();
+    });
+  });
+});
+
+tap.test('should create a SmartStream from an Observable', async () => {
+  const observableData = 'Observable test data';
+  const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
+  
+  const smartStream = SmartStream.fromObservable(testObservable);
+  
+  let receivedData = Buffer.alloc(0);
+
+  return new Promise<void>((resolve) => {
+    smartStream.on('data', (chunk: Buffer) => {
+      receivedData = Buffer.concat([receivedData, chunk]);
+    });
+
+    smartStream.on('end', () => {
+      expect(receivedData.toString()).toEqual(observableData);
+      resolve();
+    });
+  });
+});
+
+tap.start();

test/test.streamfunction.ts

@@ -1,5 +1,5 @@
-import { expect, tap } from '@pushrocks/tapbundle';
-import * as smartfile from '@pushrocks/smartfile';
+import { expect, tap } from '@push.rocks/tapbundle';
+import * as smartfile from '@push.rocks/smartfile';
 
 import * as smartstream from '../ts/index.js';
 
@@ -33,9 +33,7 @@ tap.test('should handle a read stream', async (tools) => {
 
 tap.test('should create a valid Intake', async (tools) => {
   testIntake = new smartstream.StreamIntake<string>();
-  testIntake
-    .getReadable()
-    .pipe(
+  testIntake.pipe(
       smartstream.createDuplexStream<string, string>(
         async (chunkString) => {
           await tools.delayFor(100);

test/test.ts

@@ -1,13 +1,13 @@
-import fs from 'fs';
-import { expect, tap } from '@pushrocks/tapbundle';
+import * as smartfile from '@push.rocks/smartfile';
+import { expect, tap } from '@push.rocks/tapbundle';
 
 import * as smartstream from '../ts/smartstream.classes.streamwrapper.js';
 
 let testSmartstream: smartstream.StreamWrapper;
 tap.test('should combine a stream', async () => {
   testSmartstream = new smartstream.StreamWrapper([
-    fs.createReadStream('./test/assets/test.md'),
-    fs.createWriteStream('./test/assets/testCopy.md'),
+    smartfile.fsStream.createReadStream('./test/assets/test.md'),
+    smartfile.fsStream.createWriteStream('./test/assets/testCopy.md'),
   ]);
   await testSmartstream.run();
 });

ts/00_commitinfo_data.ts

@@ -2,7 +2,7 @@
  * autocreated commitinfo by @pushrocks/commitinfo
  */
 export const commitinfo = {
-  name: '@pushrocks/smartstream',
-  version: '2.0.2',
+  name: '@push.rocks/smartstream',
+  version: '3.0.0',
   description: 'simplifies access to node streams'
 }

ts/index.ts

@@ -1,3 +1,5 @@
+export * from './smartstream.classes.passthrough.js';
+export * from './smartstream.classes.smartstream.js';
 export * from './smartstream.classes.streamwrapper.js';
 export * from './smartstream.classes.streamintake.js';
 export * from './smartstream.duplex.js';

ts/smartstream.classes.passthrough.ts

@@ -0,0 +1,19 @@
+import * as plugins from './smartstream.plugins.js';
+
+export class PassThrough extends plugins.stream.Duplex {
+  constructor(options?: plugins.stream.DuplexOptions) {
+    super(options);
+  }
+
+  _read(size: number): void {
+    // No-op: Data written will be automatically available for reading.
+  }
+
+  _write(chunk: any, encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
+    if (this.push(chunk, encoding)) {
+      callback();
+    } else {
+      this.once('drain', callback);
+    }
+  }
+}

ts/smartstream.classes.smartstream.ts

@@ -0,0 +1,112 @@
+import * as plugins from './smartstream.plugins.js';
+import { Duplex, type DuplexOptions } from 'stream';
+export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
+  // You can add more custom options relevant to TInput and TOutput if necessary
+}
+
+export class SmartStream<TInput = any, TOutput = any> extends Duplex {
+  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
+  private asyncChunkModifier?: (chunk: TInput) => Promise<TOutput>;
+
+  constructor(options?: SmartStreamOptions<TInput, TOutput>, asyncChunkModifierArg?: (chunk: TInput) => Promise<TOutput>) {
+    super(options);
+    this.asyncChunkModifier = asyncChunkModifierArg;
+  }
+
+  // Ensure the _write method types the chunk as TInput and encodes TOutput
+  public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
+    try {
+      if (this.asyncChunkModifier) {
+        const modifiedChunk = await this.asyncChunkModifier(chunk);
+        if (!this.push(modifiedChunk)) {
+          // Handle backpressure here if necessary
+        }
+      } else {
+        if (!this.push(chunk as unknown as TOutput)) {
+          // Handle backpressure here if necessary
+        }
+      }
+      callback();
+    } catch (err) {
+      callback(err);
+    }
+  }
+
+  static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
+    const smartStream = new SmartStream(options);
+    process.nextTick(() => {
+      smartStream.push(buffer);
+      smartStream.push(null); // Signal the end of the data
+    });
+    return smartStream;
+  }
+
+  static fromObservable(observable: plugins.smartrx.rxjs.Observable<any>, options?: DuplexOptions): SmartStream {
+    const smartStream = new SmartStream(options);
+    smartStream.observableSubscription = observable.subscribe({
+      next: (data) => {
+        if (!smartStream.push(data)) {
+          // Pause the observable if the stream buffer is full
+          smartStream.observableSubscription?.unsubscribe();
+          smartStream.once('drain', () => {
+            // Resume the observable when the stream buffer is drained
+            smartStream.observableSubscription?.unsubscribe();
+            smartStream.observableSubscription = observable.subscribe(data => {
+              smartStream.push(data);
+            });
+          });
+        }
+      },
+      error: (err) => {
+        smartStream.emit('error', err);
+      },
+      complete: () => {
+        smartStream.push(null); // Signal the end of the data
+      }
+    });
+
+    return smartStream;
+  }
+
+  static fromReplaySubject(replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>, options?: DuplexOptions): SmartStream {
+    const smartStream = new SmartStream(options);
+    let isBackpressured = false;
+
+    // Subscribe to the ReplaySubject
+    const subscription = replaySubject.subscribe({
+      next: (data) => {
+        const canPush = smartStream.push(data);
+        if (!canPush) {
+          // If push returns false, pause the subscription because of backpressure
+          isBackpressured = true;
+          subscription.unsubscribe();
+        }
+      },
+      error: (err) => {
+        smartStream.emit('error', err);
+      },
+      complete: () => {
+        smartStream.push(null); // End the stream when the ReplaySubject completes
+      }
+    });
+
+    // Listen for 'drain' event to resume the subscription if it was paused
+    smartStream.on('drain', () => {
+      if (isBackpressured) {
+        isBackpressured = false;
+        // Resubscribe to the ReplaySubject since we previously paused
+        smartStream.observableSubscription = replaySubject.subscribe({
+          next: (data) => {
+            if (!smartStream.push(data)) {
+              smartStream.observableSubscription?.unsubscribe();
+              isBackpressured = true;
+            }
+          },
+          // No need to repeat error and complete handling here because it's already set up above
+        });
+      }
+    });
+
+    return smartStream;
+  }
+}

ts/smartstream.classes.streamintake.ts

@@ -1,56 +1,42 @@
 import * as plugins from './smartstream.plugins.js';
 
-export class StreamIntake<T> {
+export class StreamIntake<T> extends plugins.stream.Readable {
   private signalEndBoolean = false;
   private chunkStore: T[] = [];
-
   public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
-
   private pushedNextDeferred = plugins.smartpromise.defer();
 
-  private readableStream = plugins.from2.obj(async (size, next) => {
+  constructor(options?: plugins.stream.ReadableOptions) {
+    super({ ...options, objectMode: true }); // Ensure that we are in object mode.
+    this.pushNextObservable.push('please push next');
+  }
+
+  _read(size: number): void {
     // console.log('get next');
-    // execute without backpressure
-    while (this.chunkStore.length > 0) {
-      next(null, this.chunkStore.shift());
-    }
-    if (this.signalEndBoolean) {
-      next(null, null);
-    }
+    const pushChunk = (): void => {
+      if (this.chunkStore.length > 0) {
+        // If push returns false, then we should stop reading
+        if (!this.push(this.chunkStore.shift())) {
+          return;
+        }
+      }
 
-    // lets trigger backpressure handling
-    this.pushNextObservable.push('please push next');
-    await this.pushedNextDeferred.promise;
-    this.pushedNextDeferred = plugins.smartpromise.defer();
+      if (this.chunkStore.length === 0) {
+        if (this.signalEndBoolean) {
+          // If we're done, push null to signal the end of the stream
+          this.push(null);
+        } else {
+          // Ask for more data and wait
+          this.pushNextObservable.push('please push next');
+          this.pushedNextDeferred.promise.then(() => {
+            this.pushedNextDeferred = plugins.smartpromise.defer(); // Reset the deferred
+            pushChunk(); // Try pushing the next chunk
+          });
+        }
+      }
+    };
 
-    // execute with backpressure
-    while (this.chunkStore.length > 0) {
-      next(null, this.chunkStore.shift());
-    }
-    if (this.signalEndBoolean) {
-      next(null, null);
-    }
-  });
-
-  constructor() {
-    this.pushNextObservable.push('please push next');
-  }
-
-  /**
-   * returns a new style readble stream
-   */
-  public getReadable() {
-    const readable = new plugins.stream.Readable({
-      objectMode: true,
-    });
-    return readable.wrap(this.readableStream);
-  }
-
-  /**
-   * returns an oldstyle readble stream
-   */
-  public getReadableStream() {
-    return this.readableStream;
+    pushChunk();
   }
 
   public pushData(chunkData: T) {

ts/smartstream.plugins.ts

@@ -4,8 +4,8 @@ import * as stream from 'stream';
 export { stream };
 
 // pushrocks scope
-import * as smartpromise from '@pushrocks/smartpromise';
-import * as smartrx from '@pushrocks/smartrx';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrx from '@push.rocks/smartrx';
 
 export { smartpromise, smartrx };
 

tsconfig.json

@@ -3,7 +3,12 @@
     "experimentalDecorators": true,
     "useDefineForClassFields": false,
     "target": "ES2022",
-    "module": "ES2022",
-    "moduleResolution": "nodenext"
-  }
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "exclude": [
+    "dist_*/**/*.d.ts"
+  ]
 }