Compare commits
106 Commits
Author | SHA1 | Date | |
---|---|---|---|
d9d96b8bb7 | |||
a961eea431 | |||
edb58ade28 | |||
753a481765 | |||
bbbd1b73b9 | |||
271d0be106 | |||
0ceeacd5a0 | |||
287695e445 | |||
60f9e541a5 | |||
96ea67e135 | |||
ba0a2023ad | |||
a09c359847 | |||
e2b4d772b3 | |||
0f46b62b2d | |||
9bf37469c6 | |||
12bb125bdc | |||
703dc11c6c | |||
28725d1723 | |||
c77e0f2ba6 | |||
196fb6d396 | |||
df0ddf04b3 | |||
2e1aa4a8ff | |||
bc09033af0 | |||
22df9dfd94 | |||
d48ef6eb43 | |||
9421c652a2 | |||
a6ab15bf1d | |||
00d1455367 | |||
116a281c6c | |||
9bf6f251c4 | |||
e3427c2498 | |||
a400a0a04c | |||
91392e8bd5 | |||
d161d6613a | |||
7a14e67f4f | |||
465ccfec40 | |||
3adb16d1f8 | |||
a9230ca790 | |||
788f2665c2 | |||
7b678cc856 | |||
12c9d8cc9d | |||
3a2dc1c37e | |||
1f67bc0e1e | |||
b15ddd987c | |||
cc43080513 | |||
49d235411f | |||
d238662bea | |||
8efb2b1093 | |||
4926f57d83 | |||
86552f2b1b | |||
353a8ecde6 | |||
3e03b81a43 | |||
5e4ec5b837 | |||
62796f7151 | |||
2c1d9f05ce | |||
34cbf28972 | |||
1b6e38c040 | |||
b135e6023a | |||
91d01f3689 | |||
e8e067ea77 | |||
2cb490cd2a | |||
98397bb85e | |||
f52b0de21f | |||
1c0e5f264d | |||
8a3c653213 | |||
456ce78917 | |||
5277083097 | |||
8618ac55ef | |||
ea66d1b2fb | |||
c37f62abec | |||
2c904cc1ec | |||
d1561ad1b7 | |||
0ae3fee987 | |||
047c2bd402 | |||
9ed3de718f | |||
14530f393c | |||
15a226d30d | |||
16c5c89662 | |||
851a96c014 | |||
4ea42cb9fb | |||
41eed6423d | |||
0e067004a4 | |||
9fe222b500 | |||
05e9067a34 | |||
2aff46eb0e | |||
6aa4b86598 | |||
af30268551 | |||
e562e8f099 | |||
01f4a53b5b | |||
f42b77986f | |||
b7ef295757 | |||
2818420ee9 | |||
d759e2a562 | |||
65a97c9ee0 | |||
b036bfcb92 | |||
c0304d1d10 | |||
26880f9b71 | |||
aa7c30a096 | |||
083673f02c | |||
93bcc03e72 | |||
f8b79710be | |||
addf20995a | |||
5401e5e008 | |||
144295f7c0 | |||
1e0557b722 | |||
9b67507acd |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @shipzone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
20
.gitignore
vendored
20
.gitignore
vendored
@ -1,4 +1,20 @@
|
||||
node_modules/
|
||||
.nogit/
|
||||
|
||||
# artifacts
|
||||
coverage/
|
||||
public/
|
||||
pages/
|
||||
coverage/
|
||||
|
||||
# installs
|
||||
node_modules/
|
||||
|
||||
# caches
|
||||
.yarn/
|
||||
.cache/
|
||||
.rpt2_cache
|
||||
|
||||
# builds
|
||||
dist/
|
||||
dist_*/
|
||||
|
||||
# custom
|
158
.gitlab-ci.yml
158
.gitlab-ci.yml
@ -1,42 +1,140 @@
|
||||
image: hosttoday/ht-docker-node:npmts
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- test
|
||||
- release
|
||||
- page
|
||||
testLEGACY:
|
||||
stage: test
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
before_script:
|
||||
- npm install -g @shipzone/npmci
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci test legacy
|
||||
tags:
|
||||
- docker
|
||||
testLTS:
|
||||
stage: test
|
||||
script:
|
||||
- npmci test lts
|
||||
tags:
|
||||
- docker
|
||||
testSTABLE:
|
||||
stage: test
|
||||
script:
|
||||
- npmci test stable
|
||||
tags:
|
||||
- docker
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci publish
|
||||
- npmci git mirror
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
pages:
|
||||
image: hosttoday/ht-docker-node:npmpage
|
||||
stage: page
|
||||
- notpriv
|
||||
|
||||
auditProductionDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci command npmpage --host gitlab
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --production --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=prod --production
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
auditDevDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=dev
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
only:
|
||||
- tags
|
||||
script:
|
||||
- npmci command npm install -g typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci command npm install -g @git.zone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
- public
|
||||
allow_failure: true
|
||||
|
11
.vscode/launch.json
vendored
Normal file
11
.vscode/launch.json
vendored
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"command": "npm test",
|
||||
"name": "Run npm test",
|
||||
"request": "launch",
|
||||
"type": "node-terminal"
|
||||
}
|
||||
]
|
||||
}
|
26
.vscode/settings.json
vendored
Normal file
26
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"json.schemas": [
|
||||
{
|
||||
"fileMatch": ["/npmextra.json"],
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"npmci": {
|
||||
"type": "object",
|
||||
"description": "settings for npmci"
|
||||
},
|
||||
"gitzone": {
|
||||
"type": "object",
|
||||
"description": "settings for gitzone",
|
||||
"properties": {
|
||||
"projectType": {
|
||||
"type": "string",
|
||||
"enum": ["website", "element", "service", "npm", "wcc"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
37
README.md
37
README.md
@ -1,37 +0,0 @@
|
||||
# smartstream
|
||||
simplifies access to node streams, TypeScript ready!
|
||||
|
||||
## Availabililty
|
||||
[](https://www.npmjs.com/package/smartstream)
|
||||
[](https://gitlab.com/pushrocks/smartstream)
|
||||
[](https://github.com/pushrocks/smartstream)
|
||||
[](https://pushrocks.gitlab.io/smartstream/gitbook)
|
||||
|
||||
## Status for master
|
||||
[](https://gitlab.com/pushrocks/smartstream/commits/master)
|
||||
[](https://gitlab.com/pushrocks/smartstream/commits/master)
|
||||
[](https://david-dm.org/pushrocks/smartstream)
|
||||
[](https://www.bithound.io/github/pushrocks/smartstream/master/dependencies/npm)
|
||||
[](https://www.bithound.io/github/pushrocks/smartstream)
|
||||
[](https://nodejs.org/dist/latest-v6.x/docs/api/)
|
||||
[](https://nodejs.org/dist/latest-v6.x/docs/api/)
|
||||
[](http://standardjs.com/)
|
||||
|
||||
## Usage
|
||||
We recommend the use of TypeScript for best in class intellisense support.
|
||||
|
||||
```typescript
|
||||
import { Smartstream } from 'smartstream'
|
||||
import * as gUglify from 'gulp-uglify'
|
||||
|
||||
let mySmartstream = new Smartstream([
|
||||
gulp.src(['./file1.js','./file2.js']),
|
||||
gUglify(),
|
||||
gulp.dest('./some/output/path')
|
||||
])
|
||||
|
||||
mySmartstream.onError((err) => { /* handle error */ }) // handles all errors in stream
|
||||
myStream.onCustomEvent('myeventname', (args...) => { /* Do something */ }) // emit an custom event anywhere in your stream
|
||||
mySmartstream.run().then(() => {/* do something when stream is finished */})
|
||||
```
|
||||
|
37
dist/index.d.ts
vendored
37
dist/index.d.ts
vendored
@ -1,37 +0,0 @@
|
||||
/// <reference types="q" />
|
||||
import * as plugins from './smartstream.plugins';
|
||||
export interface IErrorFunction {
|
||||
(err: any): any;
|
||||
}
|
||||
export interface ICustomEventFunction {
|
||||
(): any;
|
||||
}
|
||||
export interface ICustomEventObject {
|
||||
eventName: string;
|
||||
eventFunction: ICustomEventFunction;
|
||||
}
|
||||
/**
|
||||
* class Smartstream handles
|
||||
*/
|
||||
export declare class Smartstream {
|
||||
private streamArray;
|
||||
private customEventObjectArray;
|
||||
private streamStartedDeferred;
|
||||
/**
|
||||
* constructor
|
||||
*/
|
||||
constructor(streamArrayArg: any[]);
|
||||
/**
|
||||
* make something with the stream itself
|
||||
*/
|
||||
streamStarted(): plugins.q.Promise<any>;
|
||||
/**
|
||||
* attach listener to custom event
|
||||
*/
|
||||
onCustomEvent(eventNameArg: string, eventFunctionArg: ICustomEventFunction): void;
|
||||
/**
|
||||
* run the stream
|
||||
* @returns Promise
|
||||
*/
|
||||
run(): plugins.q.Promise<void>;
|
||||
}
|
74
dist/index.js
vendored
74
dist/index.js
vendored
@ -1,74 +0,0 @@
|
||||
"use strict";
|
||||
const plugins = require("./smartstream.plugins");
|
||||
/**
|
||||
* class Smartstream handles
|
||||
*/
|
||||
class Smartstream {
|
||||
/**
|
||||
* constructor
|
||||
*/
|
||||
constructor(streamArrayArg) {
|
||||
this.streamArray = [];
|
||||
this.customEventObjectArray = [];
|
||||
this.streamStartedDeferred = plugins.q.defer();
|
||||
this.streamArray = streamArrayArg;
|
||||
}
|
||||
/**
|
||||
* make something with the stream itself
|
||||
*/
|
||||
streamStarted() {
|
||||
return this.streamStartedDeferred.promise;
|
||||
}
|
||||
/**
|
||||
* attach listener to custom event
|
||||
*/
|
||||
onCustomEvent(eventNameArg, eventFunctionArg) {
|
||||
this.customEventObjectArray.push({
|
||||
eventName: eventNameArg,
|
||||
eventFunction: eventFunctionArg
|
||||
});
|
||||
}
|
||||
/**
|
||||
* run the stream
|
||||
* @returns Promise
|
||||
*/
|
||||
run() {
|
||||
let done = plugins.q.defer();
|
||||
// clone Array
|
||||
let streamExecutionArray = [];
|
||||
for (let streamItem of this.streamArray) {
|
||||
streamExecutionArray.push(streamItem);
|
||||
}
|
||||
// combine the stream
|
||||
let finalStream = null;
|
||||
let firstIteration = true;
|
||||
for (let stream of streamExecutionArray) {
|
||||
if (firstIteration === true) {
|
||||
finalStream = stream;
|
||||
}
|
||||
stream.on('error', (err) => {
|
||||
done.reject(err);
|
||||
});
|
||||
for (let customEventObject of this.customEventObjectArray) {
|
||||
stream.on(customEventObject.eventName, customEventObject.eventFunction);
|
||||
}
|
||||
if (!firstIteration) {
|
||||
finalStream = finalStream.pipe(stream);
|
||||
}
|
||||
firstIteration = false;
|
||||
}
|
||||
this.streamStartedDeferred.resolve();
|
||||
finalStream.on('end', function () {
|
||||
done.resolve();
|
||||
});
|
||||
finalStream.on('close', function () {
|
||||
done.resolve();
|
||||
});
|
||||
finalStream.on('finish', function () {
|
||||
done.resolve();
|
||||
});
|
||||
return done.promise;
|
||||
}
|
||||
}
|
||||
exports.Smartstream = Smartstream;
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEsaURBQWdEO0FBZWhEOztHQUVHO0FBQ0g7SUFLSTs7T0FFRztJQUNILFlBQVksY0FBcUI7UUFQekIsZ0JBQVcsR0FBRyxFQUFFLENBQUE7UUFDaEIsMkJBQXNCLEdBQXlCLEVBQUUsQ0FBQTtRQUNqRCwwQkFBcUIsR0FBRyxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssRUFBRSxDQUFBO1FBTTdDLElBQUksQ0FBQyxXQUFXLEdBQUcsY0FBYyxDQUFBO0lBQ3JDLENBQUM7SUFFRDs7T0FFRztJQUNILGFBQWE7UUFDVCxNQUFNLENBQUMsSUFBSSxDQUFDLHFCQUFxQixDQUFDLE9BQU8sQ0FBQTtJQUM3QyxDQUFDO0lBRUQ7O09BRUc7SUFDSCxhQUFhLENBQUMsWUFBb0IsRUFBRSxnQkFBc0M7UUFDdEUsSUFBSSxDQUFDLHNCQUFzQixDQUFDLElBQUksQ0FBQztZQUM3QixTQUFTLEVBQUUsWUFBWTtZQUN2QixhQUFhLEVBQUUsZ0JBQWdCO1NBQ2xDLENBQUMsQ0FBQTtJQUNOLENBQUM7SUFFRDs7O09BR0c7SUFDSCxHQUFHO1FBQ0MsSUFBSSxJQUFJLEdBQUcsT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLEVBQVEsQ0FBQTtRQUVsQyxjQUFjO1FBQ2QsSUFBSSxvQkFBb0IsR0FBRyxFQUFFLENBQUE7UUFDN0IsR0FBRyxDQUFDLENBQUMsSUFBSSxVQUFVLElBQUksSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUM7WUFBQyxvQkFBb0IsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUE7UUFBQyxDQUFDO1FBRWxGLHFCQUFxQjtRQUNyQixJQUFJLFdBQVcsR0FBRyxJQUFJLENBQUE7UUFDdEIsSUFBSSxjQUFjLEdBQVksSUFBSSxDQUFBO1FBQ2xDLEdBQUcsQ0FBQyxDQUFDLElBQUksTUFBTSxJQUFJLG9CQUFvQixDQUFDLENBQUMsQ0FBQztZQUN0QyxFQUFFLENBQUMsQ0FBQyxjQUFjLEtBQUssSUFBSSxDQUFDLENBQUMsQ0FBQztnQkFDMUIsV0FBVyxHQUFHLE1BQU0sQ0FBQTtZQUN4QixDQUFDO1lBQ0QsTUFBTSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxHQUFHO2dCQUNuQixJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxDQUFBO1lBQ3BCLENBQUMsQ0FBQyxDQUFBO1lBQ0YsR0FBRyxDQUFDLENBQUMsSUFBSSxpQkFBaUIsSUFBSSxJQUFJLENBQUMsc0JBQXNCLENBQUMsQ0FBQyxDQUFDO2dCQUN4RCxNQUFNLENBQUMsRUFBRSxDQUFDLGlCQUFpQixDQUFDLFNBQVMsRUFBRSxpQkFBaUIsQ0FBQyxhQUFhLENBQUMsQ0FBQTtZQUMzRSxDQUFDO1lBQ0QsRUFBRSxDQUFDLENBQUMsQ0FBQyxjQUFjLENBQUMsQ0FBQyxDQUFDO2dCQUNsQixXQUFXLEdBQUcsV0FBVyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQTtZQUMxQyxDQUFDO1lBQ0QsY0FBYyxHQUFHLEtBQUssQ0FBQTtRQUMxQixDQUFDO1FBRUQsSUFBSSxDQUFDLHFCQUFxQixDQUFDLE9BQU8sRUFBRSxDQUFBO1FBRXBDLFdBQVcsQ0FBQyxFQUFFLENBQUMsS0FBSyxFQUFDO1lBQ2pCLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQTtRQUNsQixDQUFDLENBQUMsQ0FBQTtRQUNGLFdBQVcsQ0FBQyxFQUFFLENBQUMsT0FBTyxFQUFDO1lBQ25CLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQTtRQUNsQixDQUFDLENBQUMsQ0FBQTtRQUNGLFdBQVcsQ0FBQyxFQUFFLENBQUMsUUFBUSxFQUFDO1lBQ3BCLElBQUksQ0FBQyxPQUFPLEVBQUUsQ0FBQTtRQUNsQixDQUFDLENBQUMsQ0FBQTtRQUNGLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFBO0lBQ3ZCLENBQUM7Q0FDSjtBQXhFRCxrQ0F3RUMifQ==
|
2
dist/smartstream.plugins.d.ts
vendored
2
dist/smartstream.plugins.d.ts
vendored
@ -1,2 +0,0 @@
|
||||
import 'typings-global';
|
||||
export import q = require('q');
|
4
dist/smartstream.plugins.js
vendored
4
dist/smartstream.plugins.js
vendored
@ -1,4 +0,0 @@
|
||||
"use strict";
|
||||
require("typings-global");
|
||||
exports.q = require("q");
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRzdHJlYW0ucGx1Z2lucy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzL3NtYXJ0c3RyZWFtLnBsdWdpbnMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLDBCQUF1QjtBQUN2Qix5QkFBOEIifQ==
|
@ -1 +1,17 @@
|
||||
{}
|
||||
{
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
},
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartstream",
|
||||
"description": "simplifies access to node streams",
|
||||
"npmPackagename": "@push.rocks/smartstream",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
55
package.json
55
package.json
@ -1,31 +1,52 @@
|
||||
{
|
||||
"name": "smartstream",
|
||||
"version": "1.0.5",
|
||||
"description": "simplifies access to node streams, TypeScript ready!",
|
||||
"main": "dist/index.js",
|
||||
"typings": "dist/index.d.ts",
|
||||
"name": "@push.rocks/smartstream",
|
||||
"version": "3.0.34",
|
||||
"private": false,
|
||||
"description": "simplifies access to node streams",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "(npmts)"
|
||||
"test": "(tstest test/)",
|
||||
"build": "(tsbuild)",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+ssh://git@gitlab.com/pushrocks/smartstream.git"
|
||||
"url": "git+https://gitlab.com/push.rocks/smartstream.git"
|
||||
},
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://gitlab.com/pushrocks/smartstream/issues"
|
||||
"url": "https://gitlab.com/push.rocks/smartstream/issues"
|
||||
},
|
||||
"homepage": "https://gitlab.com/pushrocks/smartstream#README",
|
||||
"homepage": "https://gitlab.com/push.rocks/smartstream#readme",
|
||||
"devDependencies": {
|
||||
"@types/should": "^8.1.30",
|
||||
"npmts-g": "^5.2.8",
|
||||
"should": "^11.1.0",
|
||||
"typings-test": "^1.0.3"
|
||||
"@git.zone/tsbuild": "^2.1.72",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^1.0.88",
|
||||
"@push.rocks/smartfile": "^11.0.4",
|
||||
"@push.rocks/tapbundle": "^5.0.17",
|
||||
"@types/node": "^20.11.28"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/q": "0.x.x",
|
||||
"q": "^1.4.1",
|
||||
"typings-global": "^1.0.14"
|
||||
}
|
||||
"@push.rocks/lik": "^6.0.14",
|
||||
"@push.rocks/smartpromise": "^4.0.3",
|
||||
"@push.rocks/smartrx": "^3.0.7"
|
||||
},
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
],
|
||||
"files": [
|
||||
"ts/**/*",
|
||||
"ts_web/**/*",
|
||||
"dist/**/*",
|
||||
"dist_*/**/*",
|
||||
"dist_ts/**/*",
|
||||
"dist_ts_web/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
"npmextra.json",
|
||||
"readme.md"
|
||||
]
|
||||
}
|
||||
|
4988
pnpm-lock.yaml
generated
Normal file
4988
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
54
readme.md
Normal file
54
readme.md
Normal file
@ -0,0 +1,54 @@
|
||||
# @pushrocks/smartstream
|
||||
simplifies access to node streams
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smartstream)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smartstream)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/smartstream)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smartstream/)
|
||||
|
||||
## Status for master
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
Platform support | [](https://lossless.cloud) [](https://lossless.cloud)
|
||||
|
||||
## Usage
|
||||
|
||||
Use TypeScript for best in class instellisense.
|
||||
|
||||
```typescript
|
||||
import { Smartstream } from 'smartstream'
|
||||
import * as gUglify from 'gulp-uglify'
|
||||
|
||||
let mySmartstream = new Smartstream([
|
||||
gulp.src(['./file1.js','./file2.js']),
|
||||
gUglify(),
|
||||
gulp.dest('./some/output/path')
|
||||
])
|
||||
|
||||
mySmartstream.onError((err) => { /* handle error */ }) // handles all errors in stream
|
||||
myStream.onCustomEvent('myeventname', (args...) => { /* Do something */ }) // emit an custom event anywhere in your stream
|
||||
mySmartstream.run().then(() => {/* do something when stream is finished */})
|
||||
```
|
||||
|
||||
## Contribution
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
0
readme_instructions.md
Normal file
0
readme_instructions.md
Normal file
6210
test/assets/readabletext.txt
Normal file
6210
test/assets/readabletext.txt
Normal file
File diff suppressed because it is too large
Load Diff
47
test/assets/writabletext.txt
Normal file
47
test/assets/writabletext.txt
Normal file
@ -0,0 +1,47 @@
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi+wow
|
||||
hi
|
68
test/test.backpressure.ts
Normal file
68
test/test.backpressure.ts
Normal file
@ -0,0 +1,68 @@
|
||||
import { tap, expect } from '@push.rocks/tapbundle';
|
||||
import { SmartDuplex, type ISmartDuplexOptions, StreamWrapper } from '../ts/index.js';
|
||||
|
||||
tap.test('should run backpressure test', async (toolsArg) => {
|
||||
const done = toolsArg.defer();
|
||||
async function testBackpressure() {
|
||||
const stream1 = new SmartDuplex({
|
||||
name: 'stream1',
|
||||
objectMode: true,
|
||||
writeFunction: async (chunk, tools) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10)); // Slow processing
|
||||
console.log(`processed chunk ${chunk} in stream 1`);
|
||||
return chunk; // Fast processing
|
||||
},
|
||||
});
|
||||
const stream2 = new SmartDuplex({
|
||||
name: 'stream2',
|
||||
objectMode: true,
|
||||
writeFunction: async (chunk, tools) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20)); // Slow processing
|
||||
console.log(`processed chunk ${chunk} in stream 2`);
|
||||
await tools.push(chunk);
|
||||
// return chunk, optionally return ;
|
||||
},
|
||||
}); // This stream processes data more slowly
|
||||
const stream3 = new SmartDuplex({
|
||||
objectMode: true,
|
||||
name: 'stream3',
|
||||
writeFunction: async (chunk, tools) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200)); // Slow processing
|
||||
console.log(`processed chunk ${chunk} in stream 3`);
|
||||
},
|
||||
});
|
||||
|
||||
stream1.pipe(stream2).pipe(stream3);
|
||||
|
||||
let backpressured = false;
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const canContinue = stream1.write(`Chunk ${i}`, 'utf8');
|
||||
if (!canContinue) {
|
||||
backpressured = true;
|
||||
console.log(`Backpressure at chunk ${i}`);
|
||||
}
|
||||
}
|
||||
|
||||
stream1.end();
|
||||
|
||||
stream1.on('finish', () => {
|
||||
console.log('Stream 1 finished processing.');
|
||||
});
|
||||
stream2.on('finish', () => {
|
||||
console.log('Stream 2 finished processing.');
|
||||
});
|
||||
stream3.on('finish', () => {
|
||||
console.log('Stream 3 finished processing.');
|
||||
if (!backpressured) {
|
||||
throw new Error('No backpressure was observed.');
|
||||
} else {
|
||||
done.resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
testBackpressure();
|
||||
await done.promise;
|
||||
});
|
||||
|
||||
await tap.start();
|
1
test/test.d.ts
vendored
1
test/test.d.ts
vendored
@ -1 +0,0 @@
|
||||
import 'typings-test';
|
18
test/test.js
18
test/test.js
@ -1,18 +0,0 @@
|
||||
"use strict";
|
||||
require("typings-test");
|
||||
const fs = require("fs");
|
||||
const smartstream = require("../dist/index");
|
||||
let testSmartstream;
|
||||
describe('smartstream', function () {
|
||||
it('should combine a stream', function (done) {
|
||||
this.timeout(5000);
|
||||
testSmartstream = new smartstream.Smartstream([
|
||||
fs.createReadStream('./test/assets/test.md'),
|
||||
fs.createWriteStream('./test/assets/testCopy.md')
|
||||
]);
|
||||
testSmartstream.run().then(() => {
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVzdC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRlc3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBLHdCQUFxQjtBQUNyQix5QkFBeUI7QUFHekIsNkNBQTRDO0FBRTVDLElBQUksZUFBd0MsQ0FBQTtBQUU1QyxRQUFRLENBQUMsYUFBYSxFQUFFO0lBQ3BCLEVBQUUsQ0FBQyx5QkFBeUIsRUFBRSxVQUFTLElBQUk7UUFDdkMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQTtRQUNsQixlQUFlLEdBQUcsSUFBSSxXQUFXLENBQUMsV0FBVyxDQUFDO1lBQzFDLEVBQUUsQ0FBQyxnQkFBZ0IsQ0FBQyx1QkFBdUIsQ0FBQztZQUM1QyxFQUFFLENBQUMsaUJBQWlCLENBQUMsMkJBQTJCLENBQUM7U0FDcEQsQ0FBQyxDQUFBO1FBQ0YsZUFBZSxDQUFDLEdBQUcsRUFBRSxDQUFDLElBQUksQ0FBQztZQUN2QixJQUFJLEVBQUUsQ0FBQTtRQUNWLENBQUMsQ0FBQyxDQUFBO0lBQ04sQ0FBQyxDQUFDLENBQUE7QUFDTixDQUFDLENBQUMsQ0FBQSJ9
|
24
test/test.smartstream.ts
Normal file
24
test/test.smartstream.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import { SmartDuplex } from '../ts/smartstream.classes.smartduplex.js'; // Adjust the import to your file structure
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as fs from 'fs';
|
||||
|
||||
tap.test('should create a SmartStream from a Buffer', async () => {
|
||||
const bufferData = Buffer.from('This is a test buffer');
|
||||
const smartStream = SmartDuplex.fromBuffer(bufferData, {});
|
||||
|
||||
let receivedData = Buffer.alloc(0);
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
smartStream.on('data', (chunk: Buffer) => {
|
||||
receivedData = Buffer.concat([receivedData, chunk]);
|
||||
});
|
||||
|
||||
smartStream.on('end', () => {
|
||||
expect(receivedData.toString()).toEqual(bufferData.toString());
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
tap.start();
|
65
test/test.streamfunction.ts
Normal file
65
test/test.streamfunction.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
|
||||
import * as smartstream from '../ts/index.js';
|
||||
|
||||
let testIntake: smartstream.StreamIntake<string>;
|
||||
|
||||
tap.test('should handle a read stream', async (tools) => {
|
||||
const counter = 0;
|
||||
const streamWrapper = new smartstream.StreamWrapper([
|
||||
smartfile.fsStream.createReadStream('./test/assets/readabletext.txt'),
|
||||
new smartstream.SmartDuplex({
|
||||
writeFunction: async (chunkStringArg: Buffer, streamTools) => {
|
||||
// do something with the stream here
|
||||
const result = chunkStringArg.toString().substr(0, 100);
|
||||
streamTools.push('wow =========== \n');
|
||||
return Buffer.from(result);
|
||||
},
|
||||
finalFunction: async (tools) => {
|
||||
return Buffer.from('this is the end');
|
||||
},
|
||||
}),
|
||||
new smartstream.SmartDuplex({
|
||||
writeFunction: async (chunkStringArg) => {
|
||||
console.log(chunkStringArg.toString());
|
||||
},
|
||||
finalFunction: async (tools) => {
|
||||
tools.push(null);
|
||||
},
|
||||
})
|
||||
]);
|
||||
await streamWrapper.run();
|
||||
});
|
||||
|
||||
tap.test('should create a valid Intake', async (tools) => {
|
||||
testIntake = new smartstream.StreamIntake<string>();
|
||||
testIntake.pipe(
|
||||
new smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkStringArg: string, streamTools) => {
|
||||
await tools.delayFor(100);
|
||||
console.log(chunkStringArg);
|
||||
return chunkStringArg;
|
||||
}
|
||||
})
|
||||
)
|
||||
.pipe(smartfile.fsStream.createWriteStream('./test/assets/writabletext.txt'));
|
||||
const testFinished = tools.defer();
|
||||
let counter = 0;
|
||||
testIntake.pushNextObservable.subscribe(() => {
|
||||
if (counter < 50) {
|
||||
counter++;
|
||||
testIntake.pushData('hi');
|
||||
testIntake.pushData('+wow');
|
||||
testIntake.pushData('\n');
|
||||
} else {
|
||||
testIntake.signalEnd();
|
||||
testFinished.resolve();
|
||||
}
|
||||
});
|
||||
await testFinished.promise;
|
||||
testIntake.signalEnd();
|
||||
});
|
||||
|
||||
tap.start();
|
29
test/test.ts
29
test/test.ts
@ -1,20 +1,15 @@
|
||||
import 'typings-test'
|
||||
import fs = require('fs')
|
||||
import * as should from 'should'
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
|
||||
import * as smartstream from '../dist/index'
|
||||
import * as smartstream from '../ts/smartstream.classes.streamwrapper.js';
|
||||
|
||||
let testSmartstream: smartstream.Smartstream
|
||||
let testSmartstream: smartstream.StreamWrapper;
|
||||
tap.test('should combine a stream', async () => {
|
||||
testSmartstream = new smartstream.StreamWrapper([
|
||||
smartfile.fsStream.createReadStream('./test/assets/test.md'),
|
||||
smartfile.fsStream.createWriteStream('./test/assets/testCopy.md'),
|
||||
]);
|
||||
await testSmartstream.run();
|
||||
});
|
||||
|
||||
describe('smartstream', function() {
|
||||
it('should combine a stream', function(done){
|
||||
this.timeout(5000)
|
||||
testSmartstream = new smartstream.Smartstream([
|
||||
fs.createReadStream('./test/assets/test.md'),
|
||||
fs.createWriteStream('./test/assets/testCopy.md')
|
||||
])
|
||||
testSmartstream.run().then(() => {
|
||||
done()
|
||||
})
|
||||
})
|
||||
})
|
||||
tap.start();
|
||||
|
8
ts/00_commitinfo_data.ts
Normal file
8
ts/00_commitinfo_data.ts
Normal file
@ -0,0 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @pushrocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartstream',
|
||||
version: '3.0.34',
|
||||
description: 'simplifies access to node streams'
|
||||
}
|
94
ts/index.ts
94
ts/index.ts
@ -1,91 +1,5 @@
|
||||
import * as plugins from './smartstream.plugins'
|
||||
export * from './smartstream.classes.smartduplex.js';
|
||||
export * from './smartstream.classes.streamwrapper.js';
|
||||
export * from './smartstream.classes.streamintake.js';
|
||||
|
||||
export interface IErrorFunction {
|
||||
(err): any
|
||||
}
|
||||
|
||||
export interface ICustomEventFunction {
|
||||
(): any
|
||||
}
|
||||
|
||||
export interface ICustomEventObject {
|
||||
eventName: string
|
||||
eventFunction: ICustomEventFunction
|
||||
}
|
||||
|
||||
/**
|
||||
* class Smartstream handles
|
||||
*/
|
||||
export class Smartstream {
|
||||
private streamArray = []
|
||||
private customEventObjectArray: ICustomEventObject[] = []
|
||||
private streamStartedDeferred = plugins.q.defer()
|
||||
|
||||
/**
|
||||
* constructor
|
||||
*/
|
||||
constructor(streamArrayArg: any[]) {
|
||||
this.streamArray = streamArrayArg
|
||||
}
|
||||
|
||||
/**
|
||||
* make something with the stream itself
|
||||
*/
|
||||
streamStarted(): plugins.q.Promise<any> {
|
||||
return this.streamStartedDeferred.promise
|
||||
}
|
||||
|
||||
/**
|
||||
* attach listener to custom event
|
||||
*/
|
||||
onCustomEvent(eventNameArg: string, eventFunctionArg: ICustomEventFunction) {
|
||||
this.customEventObjectArray.push({
|
||||
eventName: eventNameArg,
|
||||
eventFunction: eventFunctionArg
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* run the stream
|
||||
* @returns Promise
|
||||
*/
|
||||
run(): plugins.q.Promise<void> {
|
||||
let done = plugins.q.defer<void>()
|
||||
|
||||
// clone Array
|
||||
let streamExecutionArray = []
|
||||
for (let streamItem of this.streamArray) { streamExecutionArray.push(streamItem) }
|
||||
|
||||
// combine the stream
|
||||
let finalStream = null
|
||||
let firstIteration: boolean = true
|
||||
for (let stream of streamExecutionArray) {
|
||||
if (firstIteration === true) {
|
||||
finalStream = stream
|
||||
}
|
||||
stream.on('error', (err) => {
|
||||
done.reject(err)
|
||||
})
|
||||
for (let customEventObject of this.customEventObjectArray) {
|
||||
stream.on(customEventObject.eventName, customEventObject.eventFunction)
|
||||
}
|
||||
if (!firstIteration) {
|
||||
finalStream = finalStream.pipe(stream)
|
||||
}
|
||||
firstIteration = false
|
||||
}
|
||||
|
||||
this.streamStartedDeferred.resolve()
|
||||
|
||||
finalStream.on('end',function(){
|
||||
done.resolve()
|
||||
})
|
||||
finalStream.on('close',function(){
|
||||
done.resolve()
|
||||
})
|
||||
finalStream.on('finish',function(){
|
||||
done.resolve()
|
||||
})
|
||||
return done.promise
|
||||
}
|
||||
}
|
||||
export * from './smartstream.functions.js'
|
||||
|
160
ts/smartstream.classes.smartduplex.ts
Normal file
160
ts/smartstream.classes.smartduplex.ts
Normal file
@ -0,0 +1,160 @@
|
||||
import * as plugins from './smartstream.plugins.js';
|
||||
import { Duplex, type DuplexOptions } from 'stream';
|
||||
|
||||
export interface IStreamTools {
|
||||
truncate: () => void;
|
||||
push: (pipeObject: any) => Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface IStreamWriteFunction<T, rT> {
|
||||
(chunkArg: T, toolsArg: IStreamTools): Promise<rT>;
|
||||
}
|
||||
|
||||
export interface IStreamFinalFunction<rT> {
|
||||
(toolsArg: IStreamTools): Promise<rT>;
|
||||
}
|
||||
|
||||
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
||||
/**
|
||||
* wether to print debug logs
|
||||
*/
|
||||
debug?: boolean;
|
||||
/**
|
||||
* the name of the stream
|
||||
*/
|
||||
name?: string;
|
||||
/**
|
||||
* a function that is being called to read more stuff from whereever to be processed by the stream
|
||||
* @returns
|
||||
*/
|
||||
readFunction?: () => Promise<void>;
|
||||
|
||||
/**
|
||||
* the write function is called for every chunk that is being written to the stream
|
||||
* it can push or return chunks (but does not have to) to be written to the readable side of the stream
|
||||
*/
|
||||
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
||||
|
||||
/**
|
||||
* a final function that is run at the end of the stream
|
||||
*/
|
||||
finalFunction?: IStreamFinalFunction<TOutput>;
|
||||
}
|
||||
|
||||
export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
||||
// STATIC
|
||||
static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex {
|
||||
const smartDuplex = new SmartDuplex(options);
|
||||
process.nextTick(() => {
|
||||
smartDuplex.push(buffer);
|
||||
smartDuplex.push(null); // Signal the end of the data
|
||||
});
|
||||
return smartDuplex;
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
|
||||
public options: ISmartDuplexOptions<TInput, TOutput>;
|
||||
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
||||
private debugLog(messageArg: string) {
|
||||
if (this.options.debug) {
|
||||
console.log(messageArg);
|
||||
}
|
||||
}
|
||||
|
||||
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
|
||||
super(Object.assign({
|
||||
highWaterMark: 1,
|
||||
}, optionsArg));
|
||||
this.options = optionsArg;
|
||||
this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(this.options.highWaterMark || 1)
|
||||
}
|
||||
|
||||
public async _read(size: number): Promise<void> {
|
||||
this.debugLog(`${this.options.name}: read was called`);
|
||||
await this.backpressuredArray.waitForItems();
|
||||
this.debugLog(`${this.options.name}: successfully waited for items.`);
|
||||
if (this.options.readFunction) {
|
||||
await this.options.readFunction();
|
||||
}
|
||||
let canPushMore = true;
|
||||
while(this.backpressuredArray.data.length > 0 && canPushMore) {
|
||||
const nextChunk = this.backpressuredArray.shift();
|
||||
canPushMore = this.push(nextChunk);
|
||||
}
|
||||
}
|
||||
|
||||
public async backpressuredPush (pushArg: TOutput) {
|
||||
const canPushMore = this.backpressuredArray.push(pushArg);
|
||||
if (!canPushMore) {
|
||||
this.debugLog(`${this.options.name}: cannot push more`);
|
||||
await this.backpressuredArray.waitForSpace();
|
||||
this.debugLog(`${this.options.name}: can push more again`);
|
||||
}
|
||||
return canPushMore;
|
||||
};
|
||||
|
||||
private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
|
||||
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
||||
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
||||
if (!this.options.writeFunction) {
|
||||
return callback(new Error('No stream function provided'));
|
||||
}
|
||||
|
||||
let isTruncated = false;
|
||||
const tools: IStreamTools = {
|
||||
truncate: () => {
|
||||
this.push(null);
|
||||
isTruncated = true;
|
||||
callback();
|
||||
},
|
||||
push: async (pushArg: TOutput) => {
|
||||
return await this.backpressuredPush(pushArg);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const writeDeferred = plugins.smartpromise.defer();
|
||||
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
||||
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
||||
if (isTruncated) {
|
||||
return;
|
||||
}
|
||||
if (modifiedChunk) {
|
||||
await tools.push(modifiedChunk);
|
||||
}
|
||||
callback();
|
||||
writeDeferred.resolve();
|
||||
writeDeferred.promise.then(() => {
|
||||
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
||||
});
|
||||
} catch (err) {
|
||||
callback(err);
|
||||
}
|
||||
}
|
||||
|
||||
public async _final(callback: (error?: Error | null) => void) {
|
||||
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
||||
if (this.options.finalFunction) {
|
||||
const tools: IStreamTools = {
|
||||
truncate: () => callback(),
|
||||
push: async (pipeObject) => {
|
||||
return this.backpressuredArray.push(pipeObject);
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const finalChunk = await this.options.finalFunction(tools);
|
||||
if (finalChunk) {
|
||||
this.backpressuredArray.push(finalChunk);
|
||||
}
|
||||
} catch (err) {
|
||||
this.backpressuredArray.push(null);
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
}
|
||||
this.backpressuredArray.push(null);
|
||||
callback();
|
||||
}
|
||||
}
|
52
ts/smartstream.classes.streamintake.ts
Normal file
52
ts/smartstream.classes.streamintake.ts
Normal file
@ -0,0 +1,52 @@
|
||||
import * as plugins from './smartstream.plugins.js';
|
||||
|
||||
export class StreamIntake<T> extends plugins.stream.Readable {
|
||||
private signalEndBoolean = false;
|
||||
private chunkStore: T[] = [];
|
||||
public pushNextObservable = new plugins.smartrx.ObservableIntake<any>();
|
||||
private pushedNextDeferred = plugins.smartpromise.defer();
|
||||
|
||||
constructor(options?: plugins.stream.ReadableOptions) {
|
||||
super({ ...options, objectMode: true }); // Ensure that we are in object mode.
|
||||
this.pushNextObservable.push('please push next');
|
||||
}
|
||||
|
||||
_read(size: number): void {
|
||||
// console.log('get next');
|
||||
const pushChunk = (): void => {
|
||||
if (this.chunkStore.length > 0) {
|
||||
// If push returns false, then we should stop reading
|
||||
if (!this.push(this.chunkStore.shift())) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.chunkStore.length === 0) {
|
||||
if (this.signalEndBoolean) {
|
||||
// If we're done, push null to signal the end of the stream
|
||||
this.push(null);
|
||||
} else {
|
||||
// Ask for more data and wait
|
||||
this.pushNextObservable.push('please push next');
|
||||
this.pushedNextDeferred.promise.then(() => {
|
||||
this.pushedNextDeferred = plugins.smartpromise.defer(); // Reset the deferred
|
||||
pushChunk(); // Try pushing the next chunk
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
pushChunk();
|
||||
}
|
||||
|
||||
public pushData(chunkData: T) {
|
||||
this.chunkStore.push(chunkData);
|
||||
this.pushedNextDeferred.resolve();
|
||||
}
|
||||
|
||||
public signalEnd() {
|
||||
this.signalEndBoolean = true;
|
||||
this.pushedNextDeferred.resolve();
|
||||
this.pushNextObservable.signalComplete();
|
||||
}
|
||||
}
|
96
ts/smartstream.classes.streamwrapper.ts
Normal file
96
ts/smartstream.classes.streamwrapper.ts
Normal file
@ -0,0 +1,96 @@
|
||||
import * as plugins from './smartstream.plugins.js';
|
||||
|
||||
// interfaces
|
||||
import { Transform } from 'stream';
|
||||
|
||||
export interface IErrorFunction {
|
||||
(err: Error): any;
|
||||
}
|
||||
|
||||
export interface ICustomEventFunction {
|
||||
(): any;
|
||||
}
|
||||
|
||||
export interface ICustomEventObject {
|
||||
eventName: string;
|
||||
eventFunction: ICustomEventFunction;
|
||||
}
|
||||
|
||||
/**
|
||||
* class Smartstream handles
|
||||
*/
|
||||
export class StreamWrapper {
|
||||
private streamArray: Array<plugins.stream.Duplex> = [];
|
||||
private customEventObjectArray: ICustomEventObject[] = [];
|
||||
private streamStartedDeferred = plugins.smartpromise.defer();
|
||||
|
||||
/**
|
||||
* constructor
|
||||
*/
|
||||
constructor(streamArrayArg: any[]) {
|
||||
this.streamArray = streamArrayArg;
|
||||
}
|
||||
|
||||
/**
|
||||
* make something with the stream itself
|
||||
*/
|
||||
streamStarted(): Promise<any> {
|
||||
return this.streamStartedDeferred.promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* attach listener to custom event
|
||||
*/
|
||||
onCustomEvent(eventNameArg: string, eventFunctionArg: ICustomEventFunction) {
|
||||
this.customEventObjectArray.push({
|
||||
eventName: eventNameArg,
|
||||
eventFunction: eventFunctionArg,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* run the stream
|
||||
* @returns Promise
|
||||
*/
|
||||
run(): Promise<void> {
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
|
||||
// clone Array
|
||||
const streamExecutionArray: Array<plugins.stream.Duplex> = [];
|
||||
for (const streamItem of this.streamArray) {
|
||||
streamExecutionArray.push(streamItem);
|
||||
}
|
||||
|
||||
// combine the stream
|
||||
let finalStream = null;
|
||||
let firstIteration: boolean = true;
|
||||
for (const stream of streamExecutionArray) {
|
||||
if (firstIteration === true) {
|
||||
finalStream = stream;
|
||||
}
|
||||
stream.on('error', (err) => {
|
||||
done.reject(err);
|
||||
});
|
||||
for (const customEventObject of this.customEventObjectArray) {
|
||||
stream.on(customEventObject.eventName, customEventObject.eventFunction);
|
||||
}
|
||||
if (!firstIteration) {
|
||||
finalStream = finalStream.pipe(stream);
|
||||
}
|
||||
firstIteration = false;
|
||||
}
|
||||
|
||||
this.streamStartedDeferred.resolve();
|
||||
|
||||
finalStream.on('end', () => {
|
||||
done.resolve();
|
||||
});
|
||||
finalStream.on('close', () => {
|
||||
done.resolve();
|
||||
});
|
||||
finalStream.on('finish', () => {
|
||||
done.resolve();
|
||||
});
|
||||
return done.promise;
|
||||
}
|
||||
}
|
30
ts/smartstream.functions.ts
Normal file
30
ts/smartstream.functions.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import { Transform, type TransformCallback, type TransformOptions } from 'stream';
|
||||
import { SmartDuplex } from './smartstream.classes.smartduplex.js';
|
||||
|
||||
export interface AsyncTransformFunction<TInput, TOutput> {
|
||||
(chunkArg: TInput): Promise<TOutput>;
|
||||
}
|
||||
|
||||
export function createTransformFunction<TInput, TOutput>(
|
||||
asyncFunction: AsyncTransformFunction<TInput, TOutput>,
|
||||
options?: TransformOptions
|
||||
): SmartDuplex {
|
||||
const smartDuplexStream = new SmartDuplex({
|
||||
...options,
|
||||
writeFunction: async (chunkArg, toolsArg) => {
|
||||
const result = await asyncFunction(chunkArg);
|
||||
return result;
|
||||
}
|
||||
});
|
||||
|
||||
return smartDuplexStream;
|
||||
}
|
||||
|
||||
export const createPassThrough = () => {
|
||||
return new SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg, toolsArg) => {
|
||||
return chunkArg;
|
||||
}
|
||||
})
|
||||
}
|
@ -1,2 +1,12 @@
|
||||
import 'typings-global'
|
||||
export import q = require('q')
|
||||
// node native
|
||||
import * as stream from 'stream';
|
||||
|
||||
export { stream };
|
||||
|
||||
// pushrocks scope
|
||||
import * as lik from '@push.rocks/lik';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
|
||||
export { lik, smartpromise, smartrx };
|
||||
|
||||
|
14
tsconfig.json
Normal file
14
tsconfig.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"experimentalDecorators": true,
|
||||
"useDefineForClassFields": false,
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true
|
||||
},
|
||||
"exclude": [
|
||||
"dist_*/**/*.d.ts"
|
||||
]
|
||||
}
|
@ -1,3 +0,0 @@
|
||||
{
|
||||
"extends": "tslint-config-standard"
|
||||
}
|
Reference in New Issue
Block a user