BREAKING CHANGE(core): update

This commit is contained in:
Philipp Kunz 2023-11-03 13:55:56 +01:00
parent 2c904cc1ec
commit c37f62abec
4 changed files with 97 additions and 56 deletions

View File

@ -33,9 +33,7 @@ tap.test('should handle a read stream', async (tools) => {
tap.test('should create a valid Intake', async (tools) => { tap.test('should create a valid Intake', async (tools) => {
testIntake = new smartstream.StreamIntake<string>(); testIntake = new smartstream.StreamIntake<string>();
testIntake testIntake.pipe(
.getReadable()
.pipe(
smartstream.createDuplexStream<string, string>( smartstream.createDuplexStream<string, string>(
async (chunkString) => { async (chunkString) => {
await tools.delayFor(100); await tools.delayFor(100);

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartstream', name: '@push.rocks/smartstream',
version: '2.0.8', version: '3.0.0',
description: 'simplifies access to node streams' description: 'simplifies access to node streams'
} }

View File

@ -1,20 +1,35 @@
import * as plugins from './smartstream.plugins.js'; import * as plugins from './smartstream.plugins.js';
import { Duplex, type DuplexOptions } from 'stream'; import { Duplex, type DuplexOptions } from 'stream';
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
// You can add more custom options relevant to TInput and TOutput if necessary
}
export class SmartStream extends Duplex { export class SmartStream<TInput = any, TOutput = any> extends Duplex {
private observableSubscription?: plugins.smartrx.rxjs.Subscription; private observableSubscription?: plugins.smartrx.rxjs.Subscription;
private asyncChunkModifier?: (chunk: TInput) => Promise<TOutput>;
constructor(options?: DuplexOptions) { constructor(options?: SmartStreamOptions<TInput, TOutput>, asyncChunkModifierArg?: (chunk: TInput) => Promise<TOutput>) {
super(options); super(options);
this.asyncChunkModifier = asyncChunkModifierArg;
} }
_read(size: number) { // Ensure the _write method types the chunk as TInput and encodes TOutput
// Implement if you need custom behavior, otherwise leave it empty public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
try {
if (this.asyncChunkModifier) {
const modifiedChunk = await this.asyncChunkModifier(chunk);
if (!this.push(modifiedChunk)) {
// Handle backpressure here if necessary
}
} else {
if (!this.push(chunk as unknown as TOutput)) {
// Handle backpressure here if necessary
}
} }
_write(chunk: any, encoding: string, callback: (error?: Error | null) => void) {
// Implement if you need custom behavior
callback(); callback();
} catch (err) {
callback(err);
}
} }
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream { static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
@ -52,4 +67,46 @@ export class SmartStream extends Duplex {
return smartStream; return smartStream;
} }
static fromReplaySubject(replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>, options?: DuplexOptions): SmartStream {
const smartStream = new SmartStream(options);
let isBackpressured = false;
// Subscribe to the ReplaySubject
const subscription = replaySubject.subscribe({
next: (data) => {
const canPush = smartStream.push(data);
if (!canPush) {
// If push returns false, pause the subscription because of backpressure
isBackpressured = true;
subscription.unsubscribe();
}
},
error: (err) => {
smartStream.emit('error', err);
},
complete: () => {
smartStream.push(null); // End the stream when the ReplaySubject completes
}
});
// Listen for 'drain' event to resume the subscription if it was paused
smartStream.on('drain', () => {
if (isBackpressured) {
isBackpressured = false;
// Resubscribe to the ReplaySubject since we previously paused
smartStream.observableSubscription = replaySubject.subscribe({
next: (data) => {
if (!smartStream.push(data)) {
smartStream.observableSubscription?.unsubscribe();
isBackpressured = true;
}
},
// No need to repeat error and complete handling here because it's already set up above
});
}
});
return smartStream;
}
} }

View File

@ -1,56 +1,42 @@
import * as plugins from './smartstream.plugins.js'; import * as plugins from './smartstream.plugins.js';
export class StreamIntake<T> { export class StreamIntake<T> extends plugins.stream.Readable {
private signalEndBoolean = false; private signalEndBoolean = false;
private chunkStore: T[] = []; private chunkStore: T[] = [];
public pushNextObservable = new plugins.smartrx.ObservableIntake<any>(); public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
private pushedNextDeferred = plugins.smartpromise.defer(); private pushedNextDeferred = plugins.smartpromise.defer();
private readableStream = plugins.from2.obj(async (size, next) => { constructor(options?: plugins.stream.ReadableOptions) {
super({ ...options, objectMode: true }); // Ensure that we are in object mode.
this.pushNextObservable.push('please push next');
}
_read(size: number): void {
// console.log('get next'); // console.log('get next');
// execute without backpressure const pushChunk = (): void => {
while (this.chunkStore.length > 0) { if (this.chunkStore.length > 0) {
next(null, this.chunkStore.shift()); // If push returns false, then we should stop reading
if (!this.push(this.chunkStore.shift())) {
return;
} }
}
if (this.chunkStore.length === 0) {
if (this.signalEndBoolean) { if (this.signalEndBoolean) {
next(null, null); // If we're done, push null to signal the end of the stream
} this.push(null);
} else {
// lets trigger backpressure handling // Ask for more data and wait
this.pushNextObservable.push('please push next'); this.pushNextObservable.push('please push next');
await this.pushedNextDeferred.promise; this.pushedNextDeferred.promise.then(() => {
this.pushedNextDeferred = plugins.smartpromise.defer(); this.pushedNextDeferred = plugins.smartpromise.defer(); // Reset the deferred
pushChunk(); // Try pushing the next chunk
// execute with backpressure
while (this.chunkStore.length > 0) {
next(null, this.chunkStore.shift());
}
if (this.signalEndBoolean) {
next(null, null);
}
}); });
constructor() {
this.pushNextObservable.push('please push next');
} }
/**
* returns a new style readble stream
*/
public getReadable() {
const readable = new plugins.stream.Readable({
objectMode: true,
});
return readable.wrap(this.readableStream);
} }
};
/** pushChunk();
* returns an oldstyle readble stream
*/
public getReadableStream() {
return this.readableStream;
} }
public pushData(chunkData: T) { public pushData(chunkData: T) {