fix(core): update

This commit is contained in:
Philipp Kunz 2021-04-19 11:49:25 +00:00
parent 5f70072f4e
commit 02e5a4abe1
8 changed files with 12102 additions and 501 deletions

View File

@ -19,23 +19,35 @@ mirror:
stage: security stage: security
script: script:
- npmci git mirror - npmci git mirror
only:
- tags
tags: tags:
- lossless - lossless
- docker - docker
- notpriv - notpriv
audit: auditProductionDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security
script:
- npmci npm prepare
- npmci command npm install --production --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=high --only=prod --production
tags:
- docker
auditDevDependencies:
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
stage: security stage: security
script: script:
- npmci npm prepare - npmci npm prepare
- npmci command npm install --ignore-scripts - npmci command npm install --ignore-scripts
- npmci command npm config set registry https://registry.npmjs.org - npmci command npm config set registry https://registry.npmjs.org
- npmci command npm audit --audit-level=moderate - npmci command npm audit --audit-level=high --only=dev
tags: tags:
- lossless
- docker - docker
- notpriv allow_failure: true
# ==================== # ====================
# test stage # test stage
@ -50,9 +62,7 @@ testStable:
- npmci npm test - npmci npm test
coverage: /\d+.?\d+?\%\s*coverage/ coverage: /\d+.?\d+?\%\s*coverage/
tags: tags:
- lossless
- docker - docker
- priv
testBuild: testBuild:
stage: test stage: test
@ -63,9 +73,7 @@ testBuild:
- npmci command npm run build - npmci command npm run build
coverage: /\d+.?\d+?\%\s*coverage/ coverage: /\d+.?\d+?\%\s*coverage/
tags: tags:
- lossless
- docker - docker
- notpriv
release: release:
stage: release stage: release
@ -85,6 +93,8 @@ release:
codequality: codequality:
stage: metadata stage: metadata
allow_failure: true allow_failure: true
only:
- tags
script: script:
- npmci command npm install -g tslint typescript - npmci command npm install -g tslint typescript
- npmci npm prepare - npmci npm prepare

View File

@ -15,7 +15,7 @@
"properties": { "properties": {
"projectType": { "projectType": {
"type": "string", "type": "string",
"enum": ["website", "element", "service", "npm"] "enum": ["website", "element", "service", "npm", "wcc"]
} }
} }
} }

12464
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -19,18 +19,24 @@
}, },
"homepage": "https://github.com/pushrocks/smartarchive#readme", "homepage": "https://github.com/pushrocks/smartarchive#readme",
"dependencies": { "dependencies": {
"@pushrocks/smartfile": "^7.0.11", "@pushrocks/smartfile": "^8.0.9",
"@pushrocks/smartpath": "^4.0.1", "@pushrocks/smartpath": "^4.0.3",
"@pushrocks/smartrequest": "^1.1.47", "@pushrocks/smartrequest": "^1.1.51",
"@pushrocks/smartrx": "^2.0.19",
"@pushrocks/smartunique": "^3.0.3", "@pushrocks/smartunique": "^3.0.3",
"@types/tar": "^4.0.3", "@pushrocks/streamfunction": "^2.0.1",
"tar": "^6.0.1" "@types/gunzip-maybe": "^1.4.0",
"@types/tar": "^4.0.4",
"@types/tar-stream": "^2.2.0",
"gunzip-maybe": "^1.4.2",
"tar": "^6.1.0",
"tar-stream": "^2.2.0"
}, },
"devDependencies": { "devDependencies": {
"@gitzone/tsbuild": "^2.1.22", "@gitzone/tsbuild": "^2.1.25",
"@gitzone/tstest": "^1.0.28", "@gitzone/tstest": "^1.0.52",
"@pushrocks/tapbundle": "^3.2.1", "@pushrocks/tapbundle": "^3.2.14",
"tslint": "^6.1.0", "tslint": "^6.1.3",
"tslint-config-prettier": "^1.18.0" "tslint-config-prettier": "^1.18.0"
}, },
"private": false, "private": false,
@ -45,5 +51,8 @@
"cli.js", "cli.js",
"npmextra.json", "npmextra.json",
"readme.md" "readme.md"
],
"browserslist": [
"last 1 chrome versions"
] ]
} }

View File

@ -8,13 +8,20 @@ work with archives
* [docs (typedoc)](https://pushrocks.gitlab.io/smartarchive/) * [docs (typedoc)](https://pushrocks.gitlab.io/smartarchive/)
## Status for master ## Status for master
[![pipeline status](https://gitlab.com/pushrocks/smartarchive/badges/master/pipeline.svg)](https://gitlab.com/pushrocks/smartarchive/commits/master)
[![coverage report](https://gitlab.com/pushrocks/smartarchive/badges/master/coverage.svg)](https://gitlab.com/pushrocks/smartarchive/commits/master) Status Category | Status Badge
[![npm downloads per month](https://img.shields.io/npm/dm/@pushrocks/smartarchive.svg)](https://www.npmjs.com/package/@pushrocks/smartarchive) -- | --
[![Known Vulnerabilities](https://snyk.io/test/npm/@pushrocks/smartarchive/badge.svg)](https://snyk.io/test/npm/@pushrocks/smartarchive) GitLab Pipelines | [![pipeline status](https://gitlab.com/pushrocks/smartarchive/badges/master/pipeline.svg)](https://lossless.cloud)
[![TypeScript](https://img.shields.io/badge/TypeScript->=%203.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/) GitLab Pipline Test Coverage | [![coverage report](https://gitlab.com/pushrocks/smartarchive/badges/master/coverage.svg)](https://lossless.cloud)
[![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/) npm | [![npm downloads per month](https://badgen.net/npm/dy/@pushrocks/smartarchive)](https://lossless.cloud)
[![JavaScript Style Guide](https://img.shields.io/badge/code%20style-prettier-ff69b4.svg)](https://prettier.io/) Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/pushrocks/smartarchive)](https://lossless.cloud)
TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@pushrocks/smartarchive)](https://lossless.cloud)
PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@pushrocks/smartarchive)](https://lossless.cloud)
BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@pushrocks/smartarchive)](https://lossless.cloud)
Platform support | [![Supports Windows 10](https://badgen.net/badge/supports%20Windows%2010/yes/green?icon=windows)](https://lossless.cloud) [![Supports Mac OS X](https://badgen.net/badge/supports%20Mac%20OS%20X/yes/green?icon=apple)](https://lossless.cloud)
## Usage ## Usage
@ -25,7 +32,7 @@ import * as smartarchive from 'smartarchive';
smartarchive smartarchive
.get({ .get({
from: 'https://example.com/example.zip', from: 'https://example.com/example.zip',
toPath: '/some/local/absolute/path' toPath: '/some/local/absolute/path',
}) })
.then(/*...*/); .then(/*...*/);
``` ```

View File

@ -7,12 +7,12 @@ import * as smartrequest from '@pushrocks/smartrequest';
const testPlugins = { const testPlugins = {
path, path,
smartfile, smartfile,
smartrequest smartrequest,
}; };
const testPaths = { const testPaths = {
nogitDir: testPlugins.path.join(__dirname, '../.nogit/'), nogitDir: testPlugins.path.join(__dirname, '../.nogit/'),
remoteDir: testPlugins.path.join(__dirname, '../.nogit/remote') remoteDir: testPlugins.path.join(__dirname, '../.nogit/remote'),
}; };
import * as smartarchive from '../ts/index'; import * as smartarchive from '../ts/index';
@ -21,7 +21,7 @@ tap.preTask('should prepare .nogit dir', async () => {
await testPlugins.smartfile.fs.ensureDir(testPaths.remoteDir); await testPlugins.smartfile.fs.ensureDir(testPaths.remoteDir);
}); });
tap.preTask('should prepare downloads', async tools => { tap.preTask('should prepare downloads', async (tools) => {
const downloadedFile: Buffer = ( const downloadedFile: Buffer = (
await testPlugins.smartrequest.getBinary( await testPlugins.smartrequest.getBinary(
'https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz' 'https://verdaccio.lossless.one/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
@ -35,7 +35,7 @@ tap.preTask('should prepare downloads', async tools => {
tap.test('should extract existing files on disk', async () => { tap.test('should extract existing files on disk', async () => {
const testSmartarchive = new smartarchive.SmartArchive(); const testSmartarchive = new smartarchive.SmartArchive();
await testSmartarchive.extractArchiveFromFilePath( await testSmartarchive.extractArchiveFromFilePathToFs(
testPlugins.path.join(testPaths.nogitDir, 'test.tgz'), testPlugins.path.join(testPaths.nogitDir, 'test.tgz'),
testPlugins.path.join(testPaths.nogitDir) testPlugins.path.join(testPaths.nogitDir)
); );
@ -43,8 +43,26 @@ tap.test('should extract existing files on disk', async () => {
tap.test('should download a package from the registry', async () => { tap.test('should download a package from the registry', async () => {
const testSmartarchive = new smartarchive.SmartArchive(); const testSmartarchive = new smartarchive.SmartArchive();
await testSmartarchive.extractArchiveFromUrl('https://verdaccio.lossless.one/@pushrocks%2fsmartfile/-/smartfile-7.0.11.tgz', testPaths.remoteDir); await testSmartarchive.extractArchiveFromUrlToFs(
'https://verdaccio.lossless.one/@pushrocks%2fsmartfile/-/smartfile-7.0.11.tgz',
testPaths.remoteDir
);
});
tap.test('should extract a package using tarStream', async () => {
const testSmartarchive = new smartarchive.SmartArchive();
const testTgzBuffer = (
await testPlugins.smartfile.Smartfile.fromFilePath(
testPlugins.path.join(testPaths.nogitDir, 'test.tgz')
)
).contentBuffer;
const extractionFileObservable = await testSmartarchive.extractArchiveFromBufferToObservable(
testTgzBuffer
);
const subscription = extractionFileObservable.subscribe(file => {
console.log(file.path);
});
}); });
tap.start(); tap.start();

View File

@ -1,5 +1,6 @@
import * as plugins from './smartarchive.plugins'; import * as plugins from './smartarchive.plugins';
import * as paths from './smartarchive.paths'; import * as paths from './smartarchive.paths';
import { extract } from 'tar';
export class SmartArchive { export class SmartArchive {
public archiveDirectory: string; public archiveDirectory: string;
@ -8,13 +9,13 @@ export class SmartArchive {
/** /**
* extracts an archive from a given url * extracts an archive from a given url
*/ */
public async extractArchiveFromUrl(urlArg: string, targetDir: string) { public async extractArchiveFromUrlToFs(urlArg: string, targetDir: string) {
const parsedPath = plugins.path.parse(urlArg); const parsedPath = plugins.path.parse(urlArg);
const uniqueFileName = plugins.smartunique.uni() + parsedPath.ext; const uniqueFileName = plugins.smartunique.uni() + parsedPath.ext;
const downloadPath = plugins.path.join(paths.nogitDir, uniqueFileName); const downloadPath = plugins.path.join(paths.nogitDir, uniqueFileName);
const downloadedArchive = (await plugins.smartrequest.getBinary(urlArg)).body; const downloadedArchive = (await plugins.smartrequest.getBinary(urlArg)).body;
await plugins.smartfile.memory.toFs(downloadedArchive, downloadPath); await plugins.smartfile.memory.toFs(downloadedArchive, downloadPath);
await this.extractArchiveFromFilePath(downloadPath, targetDir); await this.extractArchiveFromFilePathToFs(downloadPath, targetDir);
await plugins.smartfile.fs.remove(downloadPath); await plugins.smartfile.fs.remove(downloadPath);
} }
@ -23,16 +24,58 @@ export class SmartArchive {
* @param filePathArg * @param filePathArg
* @param targetDir * @param targetDir
*/ */
public async extractArchiveFromFilePath(filePathArg: string, targetDir: string) { public async extractArchiveFromFilePathToFs(filePathArg: string, targetDir: string) {
const parsedPath = plugins.path.parse(filePathArg); const parsedPath = plugins.path.parse(filePathArg);
switch (parsedPath.ext) { switch (parsedPath.ext) {
case '.tgz': case '.tgz':
console.log(`detected a .tgz archive`); console.log(`detected a .tgz archive`);
await plugins.tar.extract({ await plugins.tar.extract({
file: filePathArg, file: filePathArg,
cwd: targetDir cwd: targetDir,
}); });
break; break;
} }
} }
/**
* extracts to Observable
*/
public async extractArchiveFromBufferToObservable(
bufferArg: Buffer
): Promise<plugins.smartrx.rxjs.ReplaySubject<plugins.smartfile.Smartfile>> {
const intake = new plugins.streamfunction.Intake();
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<plugins.smartfile.Smartfile>();
const readableStream = intake.getReadableStream();
const extractPipeStop = plugins.tarStream.extract();
extractPipeStop.on('entry', (header, stream, next) => {
let fileBuffer: Buffer;
stream.on('data', (chunkArg) => {
if (!fileBuffer) {
fileBuffer = chunkArg;
} else {
fileBuffer = Buffer.concat([fileBuffer, chunkArg]);
}
});
stream.on('end', () => {
replaySubject.next(
new plugins.smartfile.Smartfile({
contentBuffer: fileBuffer,
path: header.name,
})
);
next();
});
stream.resume();
});
extractPipeStop.on('finish', () => {
replaySubject.unsubscribe();
});
// lets run the stream
readableStream
.pipe(plugins.gunzipMaybe())
.pipe(extractPipeStop);
intake.pushData(bufferArg);
intake.signalEnd();
return replaySubject;
}
} }

View File

@ -8,10 +8,14 @@ import * as smartfile from '@pushrocks/smartfile';
import * as smartpath from '@pushrocks/smartpath'; import * as smartpath from '@pushrocks/smartpath';
import * as smartrequest from '@pushrocks/smartrequest'; import * as smartrequest from '@pushrocks/smartrequest';
import * as smartunique from '@pushrocks/smartunique'; import * as smartunique from '@pushrocks/smartunique';
import * as streamfunction from '@pushrocks/streamfunction';
import * as smartrx from '@pushrocks/smartrx';
export { smartfile, smartpath, smartrequest, smartunique }; export { smartfile, smartpath, smartrequest, smartunique, streamfunction, smartrx };
// third party scope // third party scope
import gunzipMaybe from 'gunzip-maybe';
import tar from 'tar'; import tar from 'tar';
import tarStream from 'tar-stream';
export { tar }; export { gunzipMaybe, tar, tarStream };