fix(core): update

This commit is contained in:
Philipp Kunz 2023-11-03 21:32:24 +01:00
parent ea66d1b2fb
commit 8618ac55ef
7 changed files with 126 additions and 146 deletions

View File

@ -48,4 +48,3 @@ hi+wow
hi+wow hi+wow
hi+wow hi+wow
hi+wow hi+wow
noice

View File

@ -1,11 +1,11 @@
import { expect, tap } from '@push.rocks/tapbundle'; import { expect, tap } from '@push.rocks/tapbundle';
import { SmartStream } from '../ts/smartstream.classes.smartstream.js'; // Adjust the import to your file structure import { SmartDuplex } from '../ts/smartstream.classes.smartduplex.js'; // Adjust the import to your file structure
import * as smartrx from '@push.rocks/smartrx'; import * as smartrx from '@push.rocks/smartrx';
import * as fs from 'fs'; import * as fs from 'fs';
tap.test('should create a SmartStream from a Buffer', async () => { tap.test('should create a SmartStream from a Buffer', async () => {
const bufferData = Buffer.from('This is a test buffer'); const bufferData = Buffer.from('This is a test buffer');
const smartStream = SmartStream.fromBuffer(bufferData); const smartStream = SmartDuplex.fromBuffer(bufferData);
let receivedData = Buffer.alloc(0); let receivedData = Buffer.alloc(0);
@ -25,7 +25,7 @@ tap.test('should create a SmartStream from an Observable', async () => {
const observableData = 'Observable test data'; const observableData = 'Observable test data';
const testObservable = smartrx.rxjs.of(Buffer.from(observableData)); const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
const smartStream = SmartStream.fromObservable(testObservable); const smartStream = SmartDuplex.fromObservable(testObservable);
let receivedData = Buffer.alloc(0); let receivedData = Buffer.alloc(0);

View File

@ -7,43 +7,43 @@ let testIntake: smartstream.StreamIntake<string>;
tap.test('should handle a read stream', async (tools) => { tap.test('should handle a read stream', async (tools) => {
const counter = 0; const counter = 0;
const testSmartstream = new smartstream.StreamWrapper([ const streamWrapper = new smartstream.StreamWrapper([
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'), smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
smartstream.createDuplexStream<Buffer, Buffer>( new smartstream.SmartDuplex({
async (chunkStringArg: Buffer, streamTools) => { writeAndTransformFunction: async (chunkStringArg: Buffer, streamTools) => {
// do something with the stream here // do something with the stream here
const result = chunkStringArg.toString().substr(0, 100); const result = chunkStringArg.toString().substr(0, 100);
streamTools.pipeMore('wow =========== \n'); streamTools.push('wow =========== \n');
return Buffer.from(result); return Buffer.from(result);
}, },
async (tools) => { streamEndFunction: async (tools) => {
// tools.pipeMore('hey, this is the end')
return Buffer.from('this is the end'); return Buffer.from('this is the end');
}, },
{ objectMode: false } }),
), new smartstream.SmartDuplex({
smartstream.createDuplexStream<Buffer, string>(async (chunkStringArg) => { writeAndTransformFunction: async (chunkStringArg) => {
console.log(chunkStringArg.toString()); console.log(chunkStringArg.toString());
return null; },
streamEndFunction: async (tools) => {
tools.push(null);
},
}), }),
smartstream.cleanPipe(), smartstream.cleanPipe(),
]); ]);
await testSmartstream.run(); // await streamWrapper.run();
}); });
tap.test('should create a valid Intake', async (tools) => { tap.test('should create a valid Intake', async (tools) => {
testIntake = new smartstream.StreamIntake<string>(); testIntake = new smartstream.StreamIntake<string>();
testIntake.pipe( testIntake.pipe(
smartstream.createDuplexStream<string, string>( new smartstream.SmartDuplex({
async (chunkString) => { objectMode: true,
writeAndTransformFunction: async (chunkStringArg: string, streamTools) => {
await tools.delayFor(100); await tools.delayFor(100);
console.log(chunkString); console.log(chunkStringArg);
return chunkString; return chunkStringArg;
},
async () => {
return 'noice';
} }
) })
) )
.pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt')); .pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
const testFinished = tools.defer(); const testFinished = tools.defer();

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartstream', name: '@push.rocks/smartstream',
version: '3.0.0', version: '3.0.1',
description: 'simplifies access to node streams' description: 'simplifies access to node streams'
} }

View File

@ -1,5 +1,4 @@
export * from './smartstream.classes.passthrough.js'; export * from './smartstream.classes.passthrough.js';
export * from './smartstream.classes.smartstream.js'; export * from './smartstream.classes.smartduplex.js';
export * from './smartstream.classes.streamwrapper.js'; export * from './smartstream.classes.streamwrapper.js';
export * from './smartstream.classes.streamintake.js'; export * from './smartstream.classes.streamintake.js';
export * from './smartstream.duplex.js';

View File

@ -1,39 +1,30 @@
import * as plugins from './smartstream.plugins.js'; import * as plugins from './smartstream.plugins.js';
import { Duplex, type DuplexOptions } from 'stream'; import { Duplex, type DuplexOptions } from 'stream';
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
// You can add more custom options relevant to TInput and TOutput if necessary export interface IStreamTools {
truncate: () => void;
push: (pipeObject: any) => void;
} }
export class SmartStream<TInput = any, TOutput = any> extends Duplex { export interface IWriteAndTransformFunction<T, rT> {
private observableSubscription?: plugins.smartrx.rxjs.Subscription; (chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
private asyncChunkModifier?: (chunk: TInput) => Promise<TOutput>; }
constructor(options?: SmartStreamOptions<TInput, TOutput>, asyncChunkModifierArg?: (chunk: TInput) => Promise<TOutput>) { export interface IStreamEndFunction<rT> {
super(options); (toolsArg: IStreamTools): Promise<rT>;
this.asyncChunkModifier = asyncChunkModifierArg; }
}
// Ensure the _write method types the chunk as TInput and encodes TOutput export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) { readFunction?: () => Promise<void>;
try { writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
if (this.asyncChunkModifier) { streamEndFunction?: IStreamEndFunction<TOutput>;
const modifiedChunk = await this.asyncChunkModifier(chunk); // Add other custom options if necessary
if (!this.push(modifiedChunk)) { }
// Handle backpressure here if necessary
}
} else {
if (!this.push(chunk as unknown as TOutput)) {
// Handle backpressure here if necessary
}
}
callback();
} catch (err) {
callback(err);
}
}
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream { export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
const smartStream = new SmartStream(options); // STATIC
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartDuplex {
const smartStream = new SmartDuplex(options);
process.nextTick(() => { process.nextTick(() => {
smartStream.push(buffer); smartStream.push(buffer);
smartStream.push(null); // Signal the end of the data smartStream.push(null); // Signal the end of the data
@ -41,8 +32,11 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
return smartStream; return smartStream;
} }
static fromObservable(observable: plugins.smartrx.rxjs.Observable<any>, options?: DuplexOptions): SmartStream { static fromObservable(
const smartStream = new SmartStream(options); observable: plugins.smartrx.rxjs.Observable<any>,
options?: DuplexOptions
): SmartDuplex {
const smartStream = new SmartDuplex(options);
smartStream.observableSubscription = observable.subscribe({ smartStream.observableSubscription = observable.subscribe({
next: (data) => { next: (data) => {
if (!smartStream.push(data)) { if (!smartStream.push(data)) {
@ -51,7 +45,7 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
smartStream.once('drain', () => { smartStream.once('drain', () => {
// Resume the observable when the stream buffer is drained // Resume the observable when the stream buffer is drained
smartStream.observableSubscription?.unsubscribe(); smartStream.observableSubscription?.unsubscribe();
smartStream.observableSubscription = observable.subscribe(data => { smartStream.observableSubscription = observable.subscribe((data) => {
smartStream.push(data); smartStream.push(data);
}); });
}); });
@ -62,14 +56,17 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
}, },
complete: () => { complete: () => {
smartStream.push(null); // Signal the end of the data smartStream.push(null); // Signal the end of the data
} },
}); });
return smartStream; return smartStream;
} }
static fromReplaySubject(replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>, options?: DuplexOptions): SmartStream { static fromReplaySubject(
const smartStream = new SmartStream(options); replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>,
options?: DuplexOptions
): SmartDuplex {
const smartStream = new SmartDuplex(options);
let isBackpressured = false; let isBackpressured = false;
// Subscribe to the ReplaySubject // Subscribe to the ReplaySubject
@ -87,7 +84,7 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
}, },
complete: () => { complete: () => {
smartStream.push(null); // End the stream when the ReplaySubject completes smartStream.push(null); // End the stream when the ReplaySubject completes
} },
}); });
// Listen for 'drain' event to resume the subscription if it was paused // Listen for 'drain' event to resume the subscription if it was paused
@ -109,4 +106,72 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
return smartStream; return smartStream;
} }
// INSTANCE
private readFunction?: () => Promise<void>;
private writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
private streamEndFunction?: IStreamEndFunction<TOutput>;
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
constructor(optionsArg?: SmartStreamOptions<TInput, TOutput>) {
super(optionsArg);
this.readFunction = optionsArg?.readFunction;
this.writeAndTransformFunction = optionsArg?.writeAndTransformFunction;
this.streamEndFunction = optionsArg?.streamEndFunction;
}
public async _read(size: number): Promise<void> {
if (this.readFunction) {
await this.readFunction();
}
}
// Ensure the _write method types the chunk as TInput and encodes TOutput
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
if (!this.writeAndTransformFunction) {
return callback(new Error('No stream function provided'));
}
const tools: IStreamTools = {
truncate: () => {
this.push(null);
callback();
},
push: (pushArg: TOutput) => this.push(pushArg),
};
try {
const modifiedChunk = await this.writeAndTransformFunction(chunk, tools);
if (modifiedChunk) {
if (!this.push(modifiedChunk)) {
// Handle backpressure if necessary
}
}
callback();
} catch (err) {
callback(err);
}
}
public async _final(callback: (error?: Error | null) => void) {
if (this.streamEndFunction) {
const tools: IStreamTools = {
truncate: () => callback(),
push: (pipeObject) => this.push(pipeObject),
};
try {
const finalChunk = await this.streamEndFunction(tools);
if (finalChunk) {
this.push(finalChunk);
}
callback();
} catch (err) {
callback(err);
}
} else {
this.push(null),
callback();
}
}
} }

View File

@ -1,83 +0,0 @@
import * as plugins from './smartstream.plugins.js';
export interface ITruncateFunc {
(): void;
}
export interface IPipeMoreFunc {
(pipeObject: any): void;
}
export interface IStreamTools {
truncate: ITruncateFunc;
pipeMore: IPipeMoreFunc;
}
export interface IStreamFunction<T, rT> {
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
}
export interface IStreamEndFunction<rT> {
(toolsArg: IStreamTools): Promise<rT>;
}
export interface IStreamOptions {
objectMode?: boolean;
readableObjectMode?: boolean;
writableObjectMode?: boolean;
}
export let createDuplexStream = <T, rT>(
funcArg: IStreamFunction<T, rT>,
endFuncArg?: IStreamEndFunction<rT>,
optionsArg: IStreamOptions = {
objectMode: false,
readableObjectMode: true,
writableObjectMode: true,
}
) => {
return plugins.through2(
optionsArg,
function (chunk, enc, cb) {
let truncated = false;
const tools: IStreamTools = {
truncate: () => {
truncated = true;
cb(null, null);
},
pipeMore: (pipeObject) => {
this.push(pipeObject);
},
};
const asyncWrapper = async () => {
const resultChunk: rT = await funcArg(chunk, tools);
if (!truncated) {
cb(null, resultChunk);
}
};
asyncWrapper().catch((err) => {
console.log(err);
});
},
function (cb) {
const tools: IStreamTools = {
truncate: () => {
cb();
},
pipeMore: (pushArg) => {
this.push(pushArg);
},
};
const asyncWrapper = async () => {
if (endFuncArg) {
const result = await endFuncArg(tools);
this.push(result);
}
cb();
};
asyncWrapper().catch((err) => {
console.log(err);
});
}
);
};