BREAKING CHANGE(core): update

This commit is contained in:
Philipp Kunz 2023-11-03 13:55:56 +01:00
parent 2c904cc1ec
commit c37f62abec
4 changed files with 97 additions and 56 deletions

View File

@ -33,9 +33,7 @@ tap.test('should handle a read stream', async (tools) => {
tap.test('should create a valid Intake', async (tools) => {
testIntake = new smartstream.StreamIntake<string>();
testIntake
.getReadable()
.pipe(
testIntake.pipe(
smartstream.createDuplexStream<string, string>(
async (chunkString) => {
await tools.delayFor(100);

View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartstream',
version: '2.0.8',
version: '3.0.0',
description: 'simplifies access to node streams'
}

View File

@ -1,20 +1,35 @@
import * as plugins from './smartstream.plugins.js';
import { Duplex, type DuplexOptions } from 'stream';
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
// You can add more custom options relevant to TInput and TOutput if necessary
}
export class SmartStream extends Duplex {
export class SmartStream<TInput = any, TOutput = any> extends Duplex {
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
private asyncChunkModifier?: (chunk: TInput) => Promise<TOutput>;
constructor(options?: DuplexOptions) {
constructor(options?: SmartStreamOptions<TInput, TOutput>, asyncChunkModifierArg?: (chunk: TInput) => Promise<TOutput>) {
super(options);
this.asyncChunkModifier = asyncChunkModifierArg;
}
_read(size: number) {
// Implement if you need custom behavior, otherwise leave it empty
// Ensure the _write method types the chunk as TInput and encodes TOutput
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
try {
if (this.asyncChunkModifier) {
const modifiedChunk = await this.asyncChunkModifier(chunk);
if (!this.push(modifiedChunk)) {
// Handle backpressure here if necessary
}
} else {
if (!this.push(chunk as unknown as TOutput)) {
// Handle backpressure here if necessary
}
}
_write(chunk: any, encoding: string, callback: (error?: Error | null) => void) {
// Implement if you need custom behavior
callback();
} catch (err) {
callback(err);
}
}
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
@ -52,4 +67,46 @@ export class SmartStream extends Duplex {
return smartStream;
}
static fromReplaySubject(replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>, options?: DuplexOptions): SmartStream {
const smartStream = new SmartStream(options);
let isBackpressured = false;
// Subscribe to the ReplaySubject
const subscription = replaySubject.subscribe({
next: (data) => {
const canPush = smartStream.push(data);
if (!canPush) {
// If push returns false, pause the subscription because of backpressure
isBackpressured = true;
subscription.unsubscribe();
}
},
error: (err) => {
smartStream.emit('error', err);
},
complete: () => {
smartStream.push(null); // End the stream when the ReplaySubject completes
}
});
// Listen for 'drain' event to resume the subscription if it was paused
smartStream.on('drain', () => {
if (isBackpressured) {
isBackpressured = false;
// Resubscribe to the ReplaySubject since we previously paused
smartStream.observableSubscription = replaySubject.subscribe({
next: (data) => {
if (!smartStream.push(data)) {
smartStream.observableSubscription?.unsubscribe();
isBackpressured = true;
}
},
// No need to repeat error and complete handling here because it's already set up above
});
}
});
return smartStream;
}
}

View File

@ -1,56 +1,42 @@
import * as plugins from './smartstream.plugins.js';
export class StreamIntake<T> {
export class StreamIntake<T> extends plugins.stream.Readable {
private signalEndBoolean = false;
private chunkStore: T[] = [];
public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
private pushedNextDeferred = plugins.smartpromise.defer();
private readableStream = plugins.from2.obj(async (size, next) => {
constructor(options?: plugins.stream.ReadableOptions) {
super({ ...options, objectMode: true }); // Ensure that we are in object mode.
this.pushNextObservable.push('please push next');
}
_read(size: number): void {
// console.log('get next');
// execute without backpressure
while (this.chunkStore.length > 0) {
next(null, this.chunkStore.shift());
const pushChunk = (): void => {
if (this.chunkStore.length > 0) {
// If push returns false, then we should stop reading
if (!this.push(this.chunkStore.shift())) {
return;
}
}
if (this.chunkStore.length === 0) {
if (this.signalEndBoolean) {
next(null, null);
}
// lets trigger backpressure handling
// If we're done, push null to signal the end of the stream
this.push(null);
} else {
// Ask for more data and wait
this.pushNextObservable.push('please push next');
await this.pushedNextDeferred.promise;
this.pushedNextDeferred = plugins.smartpromise.defer();
// execute with backpressure
while (this.chunkStore.length > 0) {
next(null, this.chunkStore.shift());
}
if (this.signalEndBoolean) {
next(null, null);
}
this.pushedNextDeferred.promise.then(() => {
this.pushedNextDeferred = plugins.smartpromise.defer(); // Reset the deferred
pushChunk(); // Try pushing the next chunk
});
constructor() {
this.pushNextObservable.push('please push next');
}
/**
* returns a new style readble stream
*/
public getReadable() {
const readable = new plugins.stream.Readable({
objectMode: true,
});
return readable.wrap(this.readableStream);
}
};
/**
* returns an oldstyle readble stream
*/
public getReadableStream() {
return this.readableStream;
pushChunk();
}
public pushData(chunkData: T) {