Compare commits
18 Commits
Author | SHA1 | Date | |
---|---|---|---|
1f67bc0e1e | |||
b15ddd987c | |||
cc43080513 | |||
49d235411f | |||
d238662bea | |||
8efb2b1093 | |||
4926f57d83 | |||
86552f2b1b | |||
353a8ecde6 | |||
3e03b81a43 | |||
5e4ec5b837 | |||
62796f7151 | |||
2c1d9f05ce | |||
34cbf28972 | |||
1b6e38c040 | |||
b135e6023a | |||
91d01f3689 | |||
e8e067ea77 |
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@push.rocks/smartstream",
|
"name": "@push.rocks/smartstream",
|
||||||
"version": "3.0.4",
|
"version": "3.0.13",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "simplifies access to node streams",
|
"description": "simplifies access to node streams",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
|
0
readme_instructions.md
Normal file
0
readme_instructions.md
Normal file
@ -10,21 +10,21 @@ tap.test('should handle a read stream', async (tools) => {
|
|||||||
const streamWrapper = new smartstream.StreamWrapper([
|
const streamWrapper = new smartstream.StreamWrapper([
|
||||||
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
|
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
|
||||||
new smartstream.SmartDuplex({
|
new smartstream.SmartDuplex({
|
||||||
writeAndTransformFunction: async (chunkStringArg: Buffer, streamTools) => {
|
writeFunction: async (chunkStringArg: Buffer, streamTools) => {
|
||||||
// do something with the stream here
|
// do something with the stream here
|
||||||
const result = chunkStringArg.toString().substr(0, 100);
|
const result = chunkStringArg.toString().substr(0, 100);
|
||||||
streamTools.push('wow =========== \n');
|
streamTools.push('wow =========== \n');
|
||||||
return Buffer.from(result);
|
return Buffer.from(result);
|
||||||
},
|
},
|
||||||
streamEndFunction: async (tools) => {
|
finalFunction: async (tools) => {
|
||||||
return Buffer.from('this is the end');
|
return Buffer.from('this is the end');
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
new smartstream.SmartDuplex({
|
new smartstream.SmartDuplex({
|
||||||
writeAndTransformFunction: async (chunkStringArg) => {
|
writeFunction: async (chunkStringArg) => {
|
||||||
console.log(chunkStringArg.toString());
|
console.log(chunkStringArg.toString());
|
||||||
},
|
},
|
||||||
streamEndFunction: async (tools) => {
|
finalFunction: async (tools) => {
|
||||||
tools.push(null);
|
tools.push(null);
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@ -37,7 +37,7 @@ tap.test('should create a valid Intake', async (tools) => {
|
|||||||
testIntake.pipe(
|
testIntake.pipe(
|
||||||
new smartstream.SmartDuplex({
|
new smartstream.SmartDuplex({
|
||||||
objectMode: true,
|
objectMode: true,
|
||||||
writeAndTransformFunction: async (chunkStringArg: string, streamTools) => {
|
writeFunction: async (chunkStringArg: string, streamTools) => {
|
||||||
await tools.delayFor(100);
|
await tools.delayFor(100);
|
||||||
console.log(chunkStringArg);
|
console.log(chunkStringArg);
|
||||||
return chunkStringArg;
|
return chunkStringArg;
|
||||||
|
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartstream',
|
name: '@push.rocks/smartstream',
|
||||||
version: '3.0.4',
|
version: '3.0.13',
|
||||||
description: 'simplifies access to node streams'
|
description: 'simplifies access to node streams'
|
||||||
}
|
}
|
||||||
|
@ -2,3 +2,5 @@ export * from './smartstream.classes.passthrough.js';
|
|||||||
export * from './smartstream.classes.smartduplex.js';
|
export * from './smartstream.classes.smartduplex.js';
|
||||||
export * from './smartstream.classes.streamwrapper.js';
|
export * from './smartstream.classes.streamwrapper.js';
|
||||||
export * from './smartstream.classes.streamintake.js';
|
export * from './smartstream.classes.streamintake.js';
|
||||||
|
|
||||||
|
export * from './smartstream.functions.js'
|
||||||
|
@ -13,7 +13,9 @@ export class PassThrough extends plugins.stream.Duplex {
|
|||||||
if (this.push(chunk, encoding)) {
|
if (this.push(chunk, encoding)) {
|
||||||
callback();
|
callback();
|
||||||
} else {
|
} else {
|
||||||
this.once('drain', callback);
|
this.once('drain', () => {
|
||||||
|
callback();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -6,18 +6,18 @@ export interface IStreamTools {
|
|||||||
push: (pipeObject: any) => void;
|
push: (pipeObject: any) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IWriteAndTransformFunction<T, rT> {
|
export interface IStreamWriteFunction<T, rT> {
|
||||||
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
|
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IStreamEndFunction<rT> {
|
export interface IStreamFinalFunction<rT> {
|
||||||
(toolsArg: IStreamTools): Promise<rT>;
|
(toolsArg: IStreamTools): Promise<rT>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
|
export interface SmartStreamOptions<TInput, TOutput> extends DuplexOptions {
|
||||||
readFunction?: () => Promise<void>;
|
readFunction?: () => Promise<void>;
|
||||||
writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
||||||
streamEndFunction?: IStreamEndFunction<TOutput>;
|
finalFunction?: IStreamFinalFunction<TOutput>;
|
||||||
// Add other custom options if necessary
|
// Add other custom options if necessary
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,15 +109,15 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|||||||
|
|
||||||
// INSTANCE
|
// INSTANCE
|
||||||
private readFunction?: () => Promise<void>;
|
private readFunction?: () => Promise<void>;
|
||||||
private writeAndTransformFunction?: IWriteAndTransformFunction<TInput, TOutput>;
|
private writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
||||||
private streamEndFunction?: IStreamEndFunction<TOutput>;
|
private finalFunction?: IStreamFinalFunction<TOutput>;
|
||||||
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
||||||
|
|
||||||
constructor(optionsArg?: SmartStreamOptions<TInput, TOutput>) {
|
constructor(optionsArg?: SmartStreamOptions<TInput, TOutput>) {
|
||||||
super(optionsArg);
|
super(optionsArg);
|
||||||
this.readFunction = optionsArg?.readFunction;
|
this.readFunction = optionsArg?.readFunction;
|
||||||
this.writeAndTransformFunction = optionsArg?.writeAndTransformFunction;
|
this.writeFunction = optionsArg?.writeFunction;
|
||||||
this.streamEndFunction = optionsArg?.streamEndFunction;
|
this.finalFunction = optionsArg?.finalFunction;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async _read(size: number): Promise<void> {
|
public async _read(size: number): Promise<void> {
|
||||||
@ -128,7 +128,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|||||||
|
|
||||||
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
||||||
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
||||||
if (!this.writeAndTransformFunction) {
|
if (!this.writeFunction) {
|
||||||
return callback(new Error('No stream function provided'));
|
return callback(new Error('No stream function provided'));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -141,7 +141,7 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const modifiedChunk = await this.writeAndTransformFunction(chunk, tools);
|
const modifiedChunk = await this.writeFunction(chunk, tools);
|
||||||
if (modifiedChunk) {
|
if (modifiedChunk) {
|
||||||
if (!this.push(modifiedChunk)) {
|
if (!this.push(modifiedChunk)) {
|
||||||
// Handle backpressure if necessary
|
// Handle backpressure if necessary
|
||||||
@ -154,24 +154,24 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async _final(callback: (error?: Error | null) => void) {
|
public async _final(callback: (error?: Error | null) => void) {
|
||||||
if (this.streamEndFunction) {
|
if (this.finalFunction) {
|
||||||
const tools: IStreamTools = {
|
const tools: IStreamTools = {
|
||||||
truncate: () => callback(),
|
truncate: () => callback(),
|
||||||
push: (pipeObject) => this.push(pipeObject),
|
push: (pipeObject) => this.push(pipeObject),
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const finalChunk = await this.streamEndFunction(tools);
|
const finalChunk = await this.finalFunction(tools);
|
||||||
if (finalChunk) {
|
if (finalChunk) {
|
||||||
this.push(finalChunk);
|
this.push(finalChunk);
|
||||||
}
|
}
|
||||||
callback();
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
callback(err);
|
callback(err);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
this.push(null),
|
// nothing here
|
||||||
|
}
|
||||||
|
this.push(null);
|
||||||
callback();
|
callback();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
26
ts/smartstream.functions.ts
Normal file
26
ts/smartstream.functions.ts
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
import { Transform, type TransformCallback, type TransformOptions } from 'stream';
|
||||||
|
|
||||||
|
export interface AsyncTransformFunction<TInput, TOutput> {
|
||||||
|
(chunkArg: TInput): Promise<TOutput>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createTransformFunction<TInput, TOutput>(
|
||||||
|
asyncFunction: AsyncTransformFunction<TInput, TOutput>,
|
||||||
|
options?: TransformOptions
|
||||||
|
): Transform {
|
||||||
|
const transformStream = new Transform({
|
||||||
|
...options,
|
||||||
|
objectMode: true, // Ensure we operate in object mode
|
||||||
|
async transform(chunk: TInput, encoding: string, callback: TransformCallback) {
|
||||||
|
try {
|
||||||
|
const transformed = await asyncFunction(chunk);
|
||||||
|
this.push(transformed);
|
||||||
|
callback();
|
||||||
|
} catch (error) {
|
||||||
|
callback(error instanceof Error ? error : new Error(String(error)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return transformStream;
|
||||||
|
}
|
Reference in New Issue
Block a user