fix(core): update
This commit is contained in:
parent
ea66d1b2fb
commit
8618ac55ef
@ -48,4 +48,3 @@ hi+wow
|
|||||||
hi+wow
|
hi+wow
|
||||||
hi+wow
|
hi+wow
|
||||||
hi+wow
|
hi+wow
|
||||||
noice
|
|
@ -1,11 +1,11 @@
|
|||||||
import { expect, tap } from '@push.rocks/tapbundle';
|
import { expect, tap } from '@push.rocks/tapbundle';
|
||||||
import { SmartStream } from '../ts/smartstream.classes.smartstream.js'; // Adjust the import to your file structure
|
import { SmartDuplex } from '../ts/smartstream.classes.smartduplex.js'; // Adjust the import to your file structure
|
||||||
import * as smartrx from '@push.rocks/smartrx';
|
import * as smartrx from '@push.rocks/smartrx';
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
|
||||||
tap.test('should create a SmartStream from a Buffer', async () => {
|
tap.test('should create a SmartStream from a Buffer', async () => {
|
||||||
const bufferData = Buffer.from('This is a test buffer');
|
const bufferData = Buffer.from('This is a test buffer');
|
||||||
const smartStream = SmartStream.fromBuffer(bufferData);
|
const smartStream = SmartDuplex.fromBuffer(bufferData);
|
||||||
|
|
||||||
let receivedData = Buffer.alloc(0);
|
let receivedData = Buffer.alloc(0);
|
||||||
|
|
||||||
@ -25,7 +25,7 @@ tap.test('should create a SmartStream from an Observable', async () => {
|
|||||||
const observableData = 'Observable test data';
|
const observableData = 'Observable test data';
|
||||||
const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
|
const testObservable = smartrx.rxjs.of(Buffer.from(observableData));
|
||||||
|
|
||||||
const smartStream = SmartStream.fromObservable(testObservable);
|
const smartStream = SmartDuplex.fromObservable(testObservable);
|
||||||
|
|
||||||
let receivedData = Buffer.alloc(0);
|
let receivedData = Buffer.alloc(0);
|
||||||
|
|
||||||
|
@ -7,43 +7,43 @@ let testIntake: smartstream.StreamIntake<string>;
|
|||||||
|
|
||||||
tap.test('should handle a read stream', async (tools) => {
|
tap.test('should handle a read stream', async (tools) => {
|
||||||
const counter = 0;
|
const counter = 0;
|
||||||
const testSmartstream = new smartstream.StreamWrapper([
|
const streamWrapper = new smartstream.StreamWrapper([
|
||||||
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
|
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
|
||||||
smartstream.createDuplexStream<Buffer, Buffer>(
|
new smartstream.SmartDuplex({
|
||||||
async (chunkStringArg: Buffer, streamTools) => {
|
writeAndTransformFunction: async (chunkStringArg: Buffer, streamTools) => {
|
||||||
// do something with the stream here
|
// do something with the stream here
|
||||||
const result = chunkStringArg.toString().substr(0, 100);
|
const result = chunkStringArg.toString().substr(0, 100);
|
||||||
streamTools.pipeMore('wow =========== \n');
|
streamTools.push('wow =========== \n');
|
||||||
return Buffer.from(result);
|
return Buffer.from(result);
|
||||||
},
|
},
|
||||||
async (tools) => {
|
streamEndFunction: async (tools) => {
|
||||||
// tools.pipeMore('hey, this is the end')
|
|
||||||
return Buffer.from('this is the end');
|
return Buffer.from('this is the end');
|
||||||
},
|
},
|
||||||
{ objectMode: false }
|
}),
|
||||||
),
|
new smartstream.SmartDuplex({
|
||||||
smartstream.createDuplexStream<Buffer, string>(async (chunkStringArg) => {
|
writeAndTransformFunction: async (chunkStringArg) => {
|
||||||
console.log(chunkStringArg.toString());
|
console.log(chunkStringArg.toString());
|
||||||
return null;
|
},
|
||||||
|
streamEndFunction: async (tools) => {
|
||||||
|
tools.push(null);
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
smartstream.cleanPipe(),
|
smartstream.cleanPipe(),
|
||||||
]);
|
]);
|
||||||
await testSmartstream.run();
|
// await streamWrapper.run();
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('should create a valid Intake', async (tools) => {
|
tap.test('should create a valid Intake', async (tools) => {
|
||||||
testIntake = new smartstream.StreamIntake<string>();
|
testIntake = new smartstream.StreamIntake<string>();
|
||||||
testIntake.pipe(
|
testIntake.pipe(
|
||||||
smartstream.createDuplexStream<string, string>(
|
new smartstream.SmartDuplex({
|
||||||
async (chunkString) => {
|
objectMode: true,
|
||||||
|
writeAndTransformFunction: async (chunkStringArg: string, streamTools) => {
|
||||||
await tools.delayFor(100);
|
await tools.delayFor(100);
|
||||||
console.log(chunkString);
|
console.log(chunkStringArg);
|
||||||
return chunkString;
|
return chunkStringArg;
|
||||||
},
|
|
||||||
async () => {
|
|
||||||
return 'noice';
|
|
||||||
}
|
}
|
||||||
)
|
})
|
||||||
)
|
)
|
||||||
.pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
|
.pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
|
||||||
const testFinished = tools.defer();
|
const testFinished = tools.defer();
|
||||||
|
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartstream',
|
name: '@push.rocks/smartstream',
|
||||||
version: '3.0.0',
|
version: '3.0.1',
|
||||||
description: 'simplifies access to node streams'
|
description: 'simplifies access to node streams'
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
export * from './smartstream.classes.passthrough.js';
|
export * from './smartstream.classes.passthrough.js';
|
||||||
export * from './smartstream.classes.smartstream.js';
|
export * from './smartstream.classes.smartduplex.js';
|
||||||
export * from './smartstream.classes.streamwrapper.js';
|
export * from './smartstream.classes.streamwrapper.js';
|
||||||
export * from './smartstream.classes.streamintake.js';
|
export * from './smartstream.classes.streamintake.js';
|
||||||
export * from './smartstream.duplex.js';
|
|
@ -1,39 +1,30 @@
|
|||||||
import * as plugins from './smartstream.plugins.js';
|
import * as plugins from './smartstream.plugins.js';
|
||||||
import { Duplex, type DuplexOptions } from 'stream';
|
import { Duplex, type DuplexOptions } from 'stream';
|
||||||
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
|
|
||||||
// You can add more custom options relevant to TInput and TOutput if necessary
|
export interface IStreamTools {
|
||||||
|
truncate: () => void;
|
||||||
|
push: (pipeObject: any) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
export interface IWriteAndTransformFunction<T, rT> {
|
||||||
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
|
||||||
private asyncChunkModifier?: (chunk: TInput) => Promise<TOutput>;
|
}
|
||||||
|
|
||||||
constructor(options?: SmartStreamOptions<TInput, TOutput>, asyncChunkModifierArg?: (chunk: TInput) => Promise<TOutput>) {
|
export interface IStreamEndFunction<rT> {
|
||||||
super(options);
|
(toolsArg: IStreamTools): Promise<rT>;
|
||||||
this.asyncChunkModifier = asyncChunkModifierArg;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
|
||||||
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
readFunction?: () => Promise<void>;
|
||||||
try {
|
writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
|
||||||
if (this.asyncChunkModifier) {
|
streamEndFunction?: IStreamEndFunction<TOutput>;
|
||||||
const modifiedChunk = await this.asyncChunkModifier(chunk);
|
// Add other custom options if necessary
|
||||||
if (!this.push(modifiedChunk)) {
|
}
|
||||||
// Handle backpressure here if necessary
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (!this.push(chunk as unknown as TOutput)) {
|
|
||||||
// Handle backpressure here if necessary
|
|
||||||
}
|
|
||||||
}
|
|
||||||
callback();
|
|
||||||
} catch (err) {
|
|
||||||
callback(err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartStream {
|
export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
||||||
const smartStream = new SmartStream(options);
|
// STATIC
|
||||||
|
static fromBuffer(buffer: Buffer, options?: DuplexOptions): SmartDuplex {
|
||||||
|
const smartStream = new SmartDuplex(options);
|
||||||
process.nextTick(() => {
|
process.nextTick(() => {
|
||||||
smartStream.push(buffer);
|
smartStream.push(buffer);
|
||||||
smartStream.push(null); // Signal the end of the data
|
smartStream.push(null); // Signal the end of the data
|
||||||
@ -41,8 +32,11 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
|||||||
return smartStream;
|
return smartStream;
|
||||||
}
|
}
|
||||||
|
|
||||||
static fromObservable(observable: plugins.smartrx.rxjs.Observable<any>, options?: DuplexOptions): SmartStream {
|
static fromObservable(
|
||||||
const smartStream = new SmartStream(options);
|
observable: plugins.smartrx.rxjs.Observable<any>,
|
||||||
|
options?: DuplexOptions
|
||||||
|
): SmartDuplex {
|
||||||
|
const smartStream = new SmartDuplex(options);
|
||||||
smartStream.observableSubscription = observable.subscribe({
|
smartStream.observableSubscription = observable.subscribe({
|
||||||
next: (data) => {
|
next: (data) => {
|
||||||
if (!smartStream.push(data)) {
|
if (!smartStream.push(data)) {
|
||||||
@ -51,7 +45,7 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
|||||||
smartStream.once('drain', () => {
|
smartStream.once('drain', () => {
|
||||||
// Resume the observable when the stream buffer is drained
|
// Resume the observable when the stream buffer is drained
|
||||||
smartStream.observableSubscription?.unsubscribe();
|
smartStream.observableSubscription?.unsubscribe();
|
||||||
smartStream.observableSubscription = observable.subscribe(data => {
|
smartStream.observableSubscription = observable.subscribe((data) => {
|
||||||
smartStream.push(data);
|
smartStream.push(data);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -62,14 +56,17 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
|||||||
},
|
},
|
||||||
complete: () => {
|
complete: () => {
|
||||||
smartStream.push(null); // Signal the end of the data
|
smartStream.push(null); // Signal the end of the data
|
||||||
}
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return smartStream;
|
return smartStream;
|
||||||
}
|
}
|
||||||
|
|
||||||
static fromReplaySubject(replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>, options?: DuplexOptions): SmartStream {
|
static fromReplaySubject(
|
||||||
const smartStream = new SmartStream(options);
|
replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>,
|
||||||
|
options?: DuplexOptions
|
||||||
|
): SmartDuplex {
|
||||||
|
const smartStream = new SmartDuplex(options);
|
||||||
let isBackpressured = false;
|
let isBackpressured = false;
|
||||||
|
|
||||||
// Subscribe to the ReplaySubject
|
// Subscribe to the ReplaySubject
|
||||||
@ -87,7 +84,7 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
|||||||
},
|
},
|
||||||
complete: () => {
|
complete: () => {
|
||||||
smartStream.push(null); // End the stream when the ReplaySubject completes
|
smartStream.push(null); // End the stream when the ReplaySubject completes
|
||||||
}
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Listen for 'drain' event to resume the subscription if it was paused
|
// Listen for 'drain' event to resume the subscription if it was paused
|
||||||
@ -109,4 +106,72 @@ export class SmartStream<TInput = any, TOutput = any> extends Duplex {
|
|||||||
|
|
||||||
return smartStream;
|
return smartStream;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// INSTANCE
|
||||||
|
private readFunction?: () => Promise<void>;
|
||||||
|
private writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
|
||||||
|
private streamEndFunction?: IStreamEndFunction<TOutput>;
|
||||||
|
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
||||||
|
|
||||||
|
constructor(optionsArg?: SmartStreamOptions<TInput, TOutput>) {
|
||||||
|
super(optionsArg);
|
||||||
|
this.readFunction = optionsArg?.readFunction;
|
||||||
|
this.writeAndTransformFunction = optionsArg?.writeAndTransformFunction;
|
||||||
|
this.streamEndFunction = optionsArg?.streamEndFunction;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async _read(size: number): Promise<void> {
|
||||||
|
if (this.readFunction) {
|
||||||
|
await this.readFunction();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
||||||
|
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
||||||
|
if (!this.writeAndTransformFunction) {
|
||||||
|
return callback(new Error('No stream function provided'));
|
||||||
|
}
|
||||||
|
|
||||||
|
const tools: IStreamTools = {
|
||||||
|
truncate: () => {
|
||||||
|
this.push(null);
|
||||||
|
callback();
|
||||||
|
},
|
||||||
|
push: (pushArg: TOutput) => this.push(pushArg),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const modifiedChunk = await this.writeAndTransformFunction(chunk, tools);
|
||||||
|
if (modifiedChunk) {
|
||||||
|
if (!this.push(modifiedChunk)) {
|
||||||
|
// Handle backpressure if necessary
|
||||||
|
}
|
||||||
|
}
|
||||||
|
callback();
|
||||||
|
} catch (err) {
|
||||||
|
callback(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async _final(callback: (error?: Error | null) => void) {
|
||||||
|
if (this.streamEndFunction) {
|
||||||
|
const tools: IStreamTools = {
|
||||||
|
truncate: () => callback(),
|
||||||
|
push: (pipeObject) => this.push(pipeObject),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const finalChunk = await this.streamEndFunction(tools);
|
||||||
|
if (finalChunk) {
|
||||||
|
this.push(finalChunk);
|
||||||
|
}
|
||||||
|
callback();
|
||||||
|
} catch (err) {
|
||||||
|
callback(err);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.push(null),
|
||||||
|
callback();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
@ -1,83 +0,0 @@
|
|||||||
import * as plugins from './smartstream.plugins.js';
|
|
||||||
|
|
||||||
export interface ITruncateFunc {
|
|
||||||
(): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IPipeMoreFunc {
|
|
||||||
(pipeObject: any): void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IStreamTools {
|
|
||||||
truncate: ITruncateFunc;
|
|
||||||
pipeMore: IPipeMoreFunc;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IStreamFunction<T, rT> {
|
|
||||||
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IStreamEndFunction<rT> {
|
|
||||||
(toolsArg: IStreamTools): Promise<rT>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IStreamOptions {
|
|
||||||
objectMode?: boolean;
|
|
||||||
readableObjectMode?: boolean;
|
|
||||||
writableObjectMode?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export let createDuplexStream = <T, rT>(
|
|
||||||
funcArg: IStreamFunction<T, rT>,
|
|
||||||
endFuncArg?: IStreamEndFunction<rT>,
|
|
||||||
optionsArg: IStreamOptions = {
|
|
||||||
objectMode: false,
|
|
||||||
readableObjectMode: true,
|
|
||||||
writableObjectMode: true,
|
|
||||||
}
|
|
||||||
) => {
|
|
||||||
return plugins.through2(
|
|
||||||
optionsArg,
|
|
||||||
function (chunk, enc, cb) {
|
|
||||||
let truncated = false;
|
|
||||||
const tools: IStreamTools = {
|
|
||||||
truncate: () => {
|
|
||||||
truncated = true;
|
|
||||||
cb(null, null);
|
|
||||||
},
|
|
||||||
pipeMore: (pipeObject) => {
|
|
||||||
this.push(pipeObject);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const asyncWrapper = async () => {
|
|
||||||
const resultChunk: rT = await funcArg(chunk, tools);
|
|
||||||
if (!truncated) {
|
|
||||||
cb(null, resultChunk);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
asyncWrapper().catch((err) => {
|
|
||||||
console.log(err);
|
|
||||||
});
|
|
||||||
},
|
|
||||||
function (cb) {
|
|
||||||
const tools: IStreamTools = {
|
|
||||||
truncate: () => {
|
|
||||||
cb();
|
|
||||||
},
|
|
||||||
pipeMore: (pushArg) => {
|
|
||||||
this.push(pushArg);
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const asyncWrapper = async () => {
|
|
||||||
if (endFuncArg) {
|
|
||||||
const result = await endFuncArg(tools);
|
|
||||||
this.push(result);
|
|
||||||
}
|
|
||||||
cb();
|
|
||||||
};
|
|
||||||
asyncWrapper().catch((err) => {
|
|
||||||
console.log(err);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
);
|
|
||||||
};
|
|
Loading…
Reference in New Issue
Block a user