Compare commits
29 Commits
Author | SHA1 | Date | |
---|---|---|---|
f99f6d96c5 | |||
b4be70f43a | |||
785e26e72d | |||
e1891a6aa3 | |||
f257c0c5a4 | |||
725546e409 | |||
b9645dfb99 | |||
b860aca103 | |||
39fb6e8ad1 | |||
04968a80b0 | |||
e4a2c143bc | |||
ed6d186a85 | |||
553c5dfe99 | |||
5d94efb9ee | |||
c978ca107b | |||
876c8ce9d8 | |||
7327bf1bd0 | |||
2dcb10d233 | |||
d53c46fa82 | |||
25e847a9ea | |||
cc0ecb3f16 | |||
2cd0846c74 | |||
49ab40af09 | |||
5ff51ff88d | |||
c578a3fdc1 | |||
ad0352a712 | |||
f921338fd6 | |||
614dae5ade | |||
f87359fb97 |
@ -6,12 +6,24 @@
|
||||
"gitzone": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smartarchive",
|
||||
"description": "work with archives",
|
||||
"description": "A library for working with archive files, providing utilities for compressing and decompressing data.",
|
||||
"npmPackagename": "@push.rocks/smartarchive",
|
||||
"license": "MIT"
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"archive",
|
||||
"compression",
|
||||
"decompression",
|
||||
"zip",
|
||||
"tar",
|
||||
"bzip2",
|
||||
"gzip"
|
||||
]
|
||||
}
|
||||
},
|
||||
"tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
}
|
||||
}
|
40
package.json
40
package.json
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@push.rocks/smartarchive",
|
||||
"version": "4.0.14",
|
||||
"description": "work with archives",
|
||||
"version": "4.0.26",
|
||||
"description": "A library for working with archive files, providing utilities for compressing and decompressing data.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
@ -12,34 +12,35 @@
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/pushrocks/smartarchive.git"
|
||||
"url": "https://code.foss.global/push.rocks/smartarchive.git"
|
||||
},
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"bugs": {
|
||||
"url": "https://github.com/pushrocks/smartarchive/issues"
|
||||
},
|
||||
"homepage": "https://github.com/pushrocks/smartarchive#readme",
|
||||
"homepage": "https://code.foss.global/push.rocks/smartarchive",
|
||||
"dependencies": {
|
||||
"@push.rocks/smartfile": "^11.0.0",
|
||||
"@push.rocks/smartpath": "^5.0.11",
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.0.20",
|
||||
"@push.rocks/smartpath": "^5.0.18",
|
||||
"@push.rocks/smartpromise": "^4.0.3",
|
||||
"@push.rocks/smartrequest": "^2.0.21",
|
||||
"@push.rocks/smartrequest": "^2.0.22",
|
||||
"@push.rocks/smartrx": "^3.0.7",
|
||||
"@push.rocks/smartstream": "^3.0.26",
|
||||
"@push.rocks/smartunique": "^3.0.6",
|
||||
"@push.rocks/smartstream": "^3.0.44",
|
||||
"@push.rocks/smartunique": "^3.0.9",
|
||||
"@push.rocks/smarturl": "^3.0.7",
|
||||
"@types/tar-stream": "^3.1.3",
|
||||
"fflate": "^0.8.1",
|
||||
"file-type": "^18.7.0",
|
||||
"tar-stream": "^3.1.6",
|
||||
"fflate": "^0.8.2",
|
||||
"file-type": "^19.0.0",
|
||||
"tar-stream": "^3.1.7",
|
||||
"through": "^2.3.8"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.1.66",
|
||||
"@git.zone/tsbuild": "^2.1.80",
|
||||
"@git.zone/tsrun": "^1.2.44",
|
||||
"@git.zone/tstest": "^1.0.84",
|
||||
"@push.rocks/tapbundle": "^5.0.15"
|
||||
"@git.zone/tstest": "^1.0.90",
|
||||
"@push.rocks/tapbundle": "^5.0.23"
|
||||
},
|
||||
"private": false,
|
||||
"files": [
|
||||
@ -56,5 +57,14 @@
|
||||
],
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
],
|
||||
"keywords": [
|
||||
"archive",
|
||||
"compression",
|
||||
"decompression",
|
||||
"zip",
|
||||
"tar",
|
||||
"bzip2",
|
||||
"gzip"
|
||||
]
|
||||
}
|
||||
|
8821
pnpm-lock.yaml
generated
8821
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
1
readme.hints.md
Normal file
1
readme.hints.md
Normal file
@ -0,0 +1 @@
|
||||
|
130
readme.md
130
readme.md
@ -1,54 +1,110 @@
|
||||
# @push.rocks/smartarchive
|
||||
work with archives
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartarchive)
|
||||
* [gitlab.com (source)](https://gitlab.com/push.rocks/smartarchive)
|
||||
* [github.com (source mirror)](https://github.com/push.rocks/smartarchive)
|
||||
* [docs (typedoc)](https://push.rocks.gitlab.io/smartarchive/)
|
||||
## Install
|
||||
|
||||
## Status for master
|
||||
To install `@push.rocks/smartarchive`, you need to use npm or yarn. Run either of the following commands in your project directory:
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
```shell
|
||||
npm install @push.rocks/smartarchive --save
|
||||
```
|
||||
|
||||
or if you prefer yarn:
|
||||
|
||||
```shell
|
||||
yarn add @push.rocks/smartarchive
|
||||
```
|
||||
|
||||
This will add `@push.rocks/smartarchive` to your project's dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
Use TypeScript for best in class instellisense.
|
||||
`@push.rocks/smartarchive` is a powerful module designed to simplify the process of working with archive files such as zip, tar, gzip, and more. It provides an easy-to-use API for extracting, creating, and analyzing archives, making it an ideal choice for projects that require manipulation of archive files.
|
||||
|
||||
```javascript
|
||||
import * as smartarchive from 'smartarchive';
|
||||
smartarchive
|
||||
.get({
|
||||
from: 'https://example.com/example.zip',
|
||||
toPath: '/some/local/absolute/path',
|
||||
})
|
||||
.then(/*...*/);
|
||||
### Getting Started
|
||||
|
||||
First, import `SmartArchive` from `@push.rocks/smartarchive` using ESM syntax:
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
```
|
||||
|
||||
For further information read the linked docs at the top of this README.
|
||||
### Extracting Archive Files
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
> | By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy.html)
|
||||
To extract an archive file, you can use `SmartArchive.fromArchiveUrl`, `SmartArchive.fromArchiveFile`, or `SmartArchive.fromArchiveStream` methods depending on the source of your archive. Here's an example of extracting an archive from a URL:
|
||||
|
||||
[](https://push.rocks)
|
||||
```typescript
|
||||
async function extractArchiveFromURL() {
|
||||
const url = 'https://example.com/archive.zip';
|
||||
const targetDir = '/path/to/extract';
|
||||
|
||||
## Contribution
|
||||
const archive = await SmartArchive.fromArchiveUrl(url);
|
||||
await archive.exportToFs(targetDir);
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
console.log('Archive extracted successfully.');
|
||||
}
|
||||
```
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
### Creating Archive Files
|
||||
|
||||
## Legal
|
||||
> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
Creating archive files such as zip or tar.gz is straightforward with `smartarchive`. At the moment, you'll prepare the contents programmatically and then compress them. Detailed support for creating archives will be covered in future updates.
|
||||
|
||||
### Analyzing Archive Files
|
||||
|
||||
Analyzing the content of archives without extracting them can be useful in various scenarios, such as when you need to inspect the archive's content before deciding to extract it. Here's how you might analyze an archive:
|
||||
|
||||
```typescript
|
||||
async function analyzeArchive() {
|
||||
const url = 'https://example.com/archive.zip';
|
||||
|
||||
const archive = await SmartArchive.fromArchiveUrl(url);
|
||||
const analysisResult = await archive.analyzeContent();
|
||||
|
||||
console.log(analysisResult); // Outputs details about the archive content
|
||||
}
|
||||
```
|
||||
|
||||
Note: Replace `analyzeContent` with the appropriate method calls as per your implementation or update, as `smartarchive` provides foundational classes and methods for interaction with archive files but does not directly implement an `analyzeContent` method by default.
|
||||
|
||||
### Stream Operations
|
||||
|
||||
`smartarchive` offers streaming operations, allowing you to work with large archives efficiently. Here's an example of using streams to extract an archive:
|
||||
|
||||
```typescript
|
||||
import { createReadStream, createWriteStream } from 'fs';
|
||||
|
||||
async function extractArchiveUsingStream() {
|
||||
const archiveStream = createReadStream('/path/to/archive.zip');
|
||||
const archive = await SmartArchive.fromArchiveStream(archiveStream);
|
||||
const extractionStream = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
extractionStream.pipe(createWriteStream('/path/to/destination'));
|
||||
}
|
||||
```
|
||||
|
||||
### Conclusion
|
||||
|
||||
`@push.rocks/smartarchive` simplifies the process of working with various archive formats in JavaScript and TypeScript projects. By providing an easy-to-use API for common archive operations, it enables developers to integrate archive manipulation features into their applications efficiently.
|
||||
|
||||
Remember, the examples provided here are to give you a starting point. Depending on your specific use case, you may need to adjust these examples to fit your project's requirements. Always refer to the latest documentation for the most current information and methods available in `@push.rocks/smartarchive`.
|
||||
|
||||
For more information and API references, check the official [`@push.rocks/smartarchive` GitHub repository](https://github.com/pushrocks/smartarchive).
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
@ -47,4 +47,4 @@ tap.skip.test('should extract a b2zip', async () => {
|
||||
);
|
||||
})
|
||||
|
||||
tap.start();
|
||||
await tap.start();
|
||||
|
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartarchive',
|
||||
version: '4.0.14',
|
||||
description: 'work with archives'
|
||||
version: '4.0.26',
|
||||
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
|
||||
}
|
||||
|
@ -49,13 +49,16 @@ export function unbzip2Stream() {
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
broken = true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
let counter = 0;
|
||||
return new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
name: 'bzip2',
|
||||
debug: false,
|
||||
highWaterMark: 1,
|
||||
writeFunction: async function (data, streamTools) {
|
||||
//console.error('received', data.length,'bytes in', typeof data);
|
||||
// console.log(`got chunk ${counter++}`)
|
||||
bufferQueue.push(data);
|
||||
hasBytes += data.length;
|
||||
if (bitReader === null) {
|
||||
@ -66,6 +69,10 @@ export function unbzip2Stream() {
|
||||
while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
|
||||
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
|
||||
const result = await decompressAndPush();
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
// console.log(result.toString());
|
||||
await streamTools.push(result);
|
||||
}
|
||||
},
|
||||
@ -73,11 +80,13 @@ export function unbzip2Stream() {
|
||||
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
|
||||
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
||||
const result = await decompressAndPush();
|
||||
streamTools.push(result);
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
await streamTools.push(result);
|
||||
}
|
||||
if (!broken) {
|
||||
if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
|
||||
this.queue(null);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
@ -4,7 +4,7 @@ import * as plugins from './plugins.js';
|
||||
export interface IAnalyzedResult {
|
||||
fileType: plugins.fileType.FileTypeResult;
|
||||
isArchive: boolean;
|
||||
resultStream: plugins.smartstream.PassThrough;
|
||||
resultStream: plugins.smartstream.SmartDuplex;
|
||||
decompressionStream: plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract;
|
||||
}
|
||||
|
||||
@ -36,37 +36,41 @@ export class ArchiveAnalyzer {
|
||||
switch (mimeTypeArg) {
|
||||
case 'application/gzip':
|
||||
return this.smartArchiveRef.gzipTools.getDecompressionStream();
|
||||
case 'application/zip':
|
||||
return this.smartArchiveRef.zipTools.getDecompressionStream();
|
||||
case 'application/x-bzip2':
|
||||
return await this.smartArchiveRef.bzip2Tools.getDecompressionStream(); // replace with your own bzip2 decompression stream
|
||||
case 'application/x-tar':
|
||||
return this.smartArchiveRef.tarTools.getDecompressionStream(); // replace with your own tar decompression stream
|
||||
default:
|
||||
// Handle unsupported formats or no decompression needed
|
||||
return new plugins.smartstream.PassThrough();
|
||||
return plugins.smartstream.createPassThrough();
|
||||
}
|
||||
}
|
||||
|
||||
public getAnalyzedStream() {
|
||||
let firstRun = true;
|
||||
const resultStream = new plugins.smartstream.PassThrough();
|
||||
const resultStream = plugins.smartstream.createPassThrough();
|
||||
const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
|
||||
readableObjectMode: true,
|
||||
writeFunction: async (chunkArg: Buffer, streamtools) => {
|
||||
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
|
||||
const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
|
||||
resultStream.push(chunkArg);
|
||||
if (firstRun) {
|
||||
firstRun = false;
|
||||
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
|
||||
const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
|
||||
/**
|
||||
* analyzed stream emits once with this object
|
||||
*/
|
||||
const result: IAnalyzedResult = {
|
||||
fileType,
|
||||
isArchive: await this.mimeTypeIsArchive(fileType?.mime),
|
||||
resultStream,
|
||||
decompressionStream,
|
||||
};
|
||||
streamtools.push(result);
|
||||
streamtools.push(null);
|
||||
return null;
|
||||
await streamtools.push(result);
|
||||
}
|
||||
await resultStream.backpressuredPush(chunkArg);
|
||||
return null;
|
||||
},
|
||||
finalFunction: async (tools) => {
|
||||
resultStream.push(null);
|
||||
|
@ -1,9 +1,10 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
|
||||
import { Bzip2Tools } from './classes.bzip2tools.js';
|
||||
import { GzipTools } from './classes.gziptools.js';
|
||||
import { TarTools } from './classes.tartools.js';
|
||||
import { Bzip2Tools } from './classes.bzip2tools.js';
|
||||
import { ZipTools } from './classes.ziptools.js';
|
||||
|
||||
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
||||
|
||||
@ -32,9 +33,10 @@ export class SmartArchive {
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
public tarTools = new TarTools(this);
|
||||
public gzipTools = new GzipTools(this);
|
||||
public bzip2Tools = new Bzip2Tools(this);
|
||||
public tarTools = new TarTools(this);
|
||||
public zipTools = new ZipTools(this);
|
||||
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
||||
|
||||
public sourceUrl: string;
|
||||
@ -77,27 +79,32 @@ export class SmartArchive {
|
||||
public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
const streamFileStream = await this.exportToStreamOfStreamFiles();
|
||||
streamFileStream.pipe(new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
console.log(chunkArg.relativeFilePath ? chunkArg.relativeFilePath : 'no relative path');
|
||||
const streamFile = chunkArg;
|
||||
const readStream = await streamFile.createReadStream();
|
||||
await plugins.smartfile.fs.ensureDir(targetDir);
|
||||
const writePath = plugins.path.join(targetDir, (streamFile.relativeFilePath || fileNameArg));
|
||||
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
|
||||
const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
|
||||
readStream.pipe(writeStream);
|
||||
writeStream.on('finish', () => {
|
||||
streamFileStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path');
|
||||
const streamFile = streamFileArg;
|
||||
const readStream = await streamFile.createReadStream();
|
||||
await plugins.smartfile.fs.ensureDir(targetDir);
|
||||
const writePath = plugins.path.join(
|
||||
targetDir,
|
||||
streamFile.relativeFilePath || fileNameArg
|
||||
);
|
||||
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
|
||||
const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
|
||||
readStream.pipe(writeStream);
|
||||
writeStream.on('finish', () => {
|
||||
done.resolve();
|
||||
});
|
||||
await done.promise;
|
||||
},
|
||||
finalFunction: async () => {
|
||||
done.resolve();
|
||||
})
|
||||
await done.promise;
|
||||
},
|
||||
finalFunction: async () => {
|
||||
done.resolve();
|
||||
}
|
||||
}));
|
||||
},
|
||||
})
|
||||
);
|
||||
return done.promise;
|
||||
}
|
||||
|
||||
@ -114,21 +121,36 @@ export class SmartArchive {
|
||||
async (analyzedResultChunk) => {
|
||||
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
|
||||
const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
|
||||
tarStream.on(
|
||||
'entry',
|
||||
async (header, stream, next) => {
|
||||
const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
|
||||
streamFileIntake.push(streamfile);
|
||||
stream.on('end', function () {
|
||||
next(); // ready for next entry
|
||||
});
|
||||
tarStream.on('entry', async (header, stream, next) => {
|
||||
if (header.type === 'directory') {
|
||||
console.log(`tar stream directory: ${header.name} ... skipping!`);
|
||||
next();
|
||||
return;
|
||||
}
|
||||
);
|
||||
console.log(`tar stream file: ${header.name}`);
|
||||
const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
|
||||
streamFileIntake.push(streamfile);
|
||||
stream.on('end', function () {
|
||||
next(); // ready for next entry
|
||||
});
|
||||
});
|
||||
tarStream.on('finish', function () {
|
||||
console.log('finished');
|
||||
streamFileIntake.signalEnd();
|
||||
});
|
||||
analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
|
||||
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
|
||||
analyzedResultChunk.resultStream
|
||||
.pipe(analyzedResultChunk.decompressionStream)
|
||||
.pipe(new plugins.smartstream.SmartDuplex({
|
||||
objectMode: true,
|
||||
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
||||
streamFileIntake.push(streamFileArg);
|
||||
},
|
||||
finalFunction: async () => {
|
||||
streamFileIntake.signalEnd();
|
||||
}
|
||||
}));
|
||||
} else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
|
||||
analyzedResultChunk.resultStream
|
||||
.pipe(analyzedResultChunk.decompressionStream)
|
||||
@ -142,8 +164,9 @@ export class SmartArchive {
|
||||
streamFileIntake.push(streamFile);
|
||||
streamFileIntake.signalEnd();
|
||||
}
|
||||
}, {
|
||||
objectMode: true
|
||||
},
|
||||
{
|
||||
objectMode: true,
|
||||
}
|
||||
);
|
||||
|
||||
|
@ -2,28 +2,116 @@ import type { SmartArchive } from './classes.smartarchive.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export class TarTools {
|
||||
// INSTANCE
|
||||
smartArchiveRef: SmartArchive;
|
||||
|
||||
constructor(smartArchiveRefArg: SmartArchive) {
|
||||
this.smartArchiveRef = smartArchiveRefArg;
|
||||
}
|
||||
|
||||
// packing
|
||||
public addFileToPack(pack: plugins.tarStream.Pack, fileName: string, content: string | Buffer) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const entry = pack.entry({ name: fileName, size: content.length }, (err: Error) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
|
||||
entry.write(content);
|
||||
entry.end();
|
||||
// packing
|
||||
public async addFileToPack(
|
||||
pack: plugins.tarStream.Pack,
|
||||
optionsArg: {
|
||||
fileName?: string;
|
||||
content?:
|
||||
| string
|
||||
| Buffer
|
||||
| plugins.smartstream.stream.Readable
|
||||
| plugins.smartfile.SmartFile
|
||||
| plugins.smartfile.StreamFile;
|
||||
byteLength?: number;
|
||||
filePath?: string;
|
||||
}
|
||||
): Promise<void> {
|
||||
return new Promise<void>(async (resolve, reject) => {
|
||||
let fileName =
|
||||
optionsArg.fileName || optionsArg.content instanceof plugins.smartfile.SmartFile
|
||||
? (optionsArg.content as plugins.smartfile.SmartFile).relative
|
||||
: null || optionsArg.content instanceof plugins.smartfile.StreamFile
|
||||
? (optionsArg.content as plugins.smartfile.StreamFile).relativeFilePath
|
||||
: null || optionsArg.filePath;
|
||||
|
||||
/**
|
||||
* contentByteLength is used to set the size of the entry in the tar file
|
||||
*/
|
||||
let contentByteLength: number;
|
||||
if (optionsArg.byteLength) {
|
||||
contentByteLength = optionsArg.byteLength;
|
||||
} else if (typeof optionsArg.content === 'string') {
|
||||
contentByteLength = Buffer.byteLength(optionsArg.content, 'utf8');
|
||||
} else if (Buffer.isBuffer(optionsArg.content)) {
|
||||
contentByteLength = optionsArg.content.length;
|
||||
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
||||
contentByteLength = await optionsArg.content.getSize(); // assuming SmartFile has getSize method
|
||||
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
||||
contentByteLength = await optionsArg.content.getSize(); // assuming StreamFile has getSize method
|
||||
} else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) {
|
||||
console.warn(
|
||||
'@push.rocks/smartarchive: When streaming, it is recommended to provide byteLength, if known.'
|
||||
);
|
||||
} else if (optionsArg.filePath) {
|
||||
const fileStat = await plugins.smartfile.fs.stat(optionsArg.filePath);
|
||||
contentByteLength = fileStat.size;
|
||||
}
|
||||
|
||||
const entry = pack.entry(
|
||||
{
|
||||
name: fileName,
|
||||
...(contentByteLength
|
||||
? {
|
||||
size: contentByteLength,
|
||||
}
|
||||
: null),
|
||||
},
|
||||
(err: Error) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
let content: plugins.smartstream.stream.Readable;
|
||||
if (Buffer.isBuffer(optionsArg.content)) {
|
||||
content = plugins.smartstream.stream.Readable.from(optionsArg.content);
|
||||
} else if (typeof optionsArg.content === 'string') {
|
||||
content = plugins.smartstream.stream.Readable.from(Buffer.from(optionsArg.content));
|
||||
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
||||
content = plugins.smartstream.stream.Readable.from(optionsArg.content.contents);
|
||||
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
||||
content = await optionsArg.content.createReadStream();
|
||||
} else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) {
|
||||
content = optionsArg.content;
|
||||
}
|
||||
|
||||
content.pipe(entry);
|
||||
entry.on('end', () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* packs a directory from disk into a tar stream
|
||||
* @param directoryPath
|
||||
*/
|
||||
public async packDirectory(directoryPath: string) {
|
||||
const fileTree = await plugins.smartfile.fs.listFileTree(directoryPath, '**/*');
|
||||
const pack = await this.getPackStream();
|
||||
for (const filePath of fileTree) {
|
||||
const absolutePath = plugins.path.join(directoryPath, filePath);
|
||||
const fileStat = await plugins.smartfile.fs.stat(absolutePath);
|
||||
await this.addFileToPack(pack, {
|
||||
byteLength: fileStat.size,
|
||||
filePath: filePath,
|
||||
fileName: filePath,
|
||||
content: plugins.smartfile.fsStream.createReadStream(absolutePath),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public async getPackStream() {
|
||||
const pack = plugins.tarStream.pack();
|
||||
return pack;
|
||||
|
77
ts/classes.ziptools.ts
Normal file
77
ts/classes.ziptools.ts
Normal file
@ -0,0 +1,77 @@
|
||||
import type { SmartArchive } from './classes.smartarchive.js';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
class DecompressZipTransform extends plugins.smartstream.SmartDuplex<ArrayBufferLike> {
|
||||
private streamtools: plugins.smartstream.IStreamTools;
|
||||
private unzipper = new plugins.fflate.Unzip(async (fileArg) => {
|
||||
let resultBuffer: Buffer;
|
||||
fileArg.ondata = async (flateError, dat, final) => {
|
||||
resultBuffer? resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)])
|
||||
: resultBuffer = Buffer.from(dat);
|
||||
if (final) {
|
||||
const streamFile = plugins.smartfile.StreamFile.fromBuffer(resultBuffer);
|
||||
streamFile.relativeFilePath = fileArg.name;
|
||||
this.streamtools.push(streamFile);
|
||||
}
|
||||
}
|
||||
fileArg.start();
|
||||
});
|
||||
constructor() {
|
||||
super({
|
||||
objectMode: true,
|
||||
writeFunction: async (chunkArg: Buffer, streamtoolsArg) => {
|
||||
this.streamtools? null : this.streamtools = streamtoolsArg;
|
||||
this.unzipper.push(chunkArg, false);
|
||||
},
|
||||
finalFunction: async () => {
|
||||
this.unzipper.push(Buffer.from(''), true);
|
||||
await plugins.smartdelay.delayFor(0);
|
||||
await this.streamtools.push(null);
|
||||
}
|
||||
});
|
||||
this.unzipper.register(plugins.fflate.UnzipInflate);
|
||||
}
|
||||
}
|
||||
|
||||
// This class wraps fflate's zip in a Node.js Transform stream for compression
|
||||
export class CompressZipTransform extends plugins.stream.Transform {
|
||||
files: { [fileName: string]: Uint8Array };
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.files = {};
|
||||
}
|
||||
|
||||
_transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) {
|
||||
// Simple example: storing chunks in memory before finalizing ZIP in _flush
|
||||
this.files['file.txt'] = new Uint8Array(chunk);
|
||||
callback();
|
||||
}
|
||||
|
||||
_flush(callback: plugins.stream.TransformCallback) {
|
||||
plugins.fflate.zip(this.files, (err, zipped) => {
|
||||
if (err) {
|
||||
callback(err);
|
||||
} else {
|
||||
this.push(Buffer.from(zipped));
|
||||
callback();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class ZipTools {
|
||||
smartArchiveRef: SmartArchive;
|
||||
|
||||
constructor(smartArchiveRefArg: SmartArchive) {
|
||||
this.smartArchiveRef = smartArchiveRefArg;
|
||||
}
|
||||
|
||||
public getCompressionStream() {
|
||||
return new CompressZipTransform();
|
||||
}
|
||||
|
||||
public getDecompressionStream() {
|
||||
return new DecompressZipTransform();
|
||||
}
|
||||
}
|
@ -6,6 +6,7 @@ export { path, stream };
|
||||
|
||||
// @pushrocks scope
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartdelay from '@push.rocks/smartdelay';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartpromise from '@push.rocks/smartpromise';
|
||||
import * as smartrequest from '@push.rocks/smartrequest';
|
||||
@ -14,7 +15,7 @@ import * as smartstream from '@push.rocks/smartstream';
|
||||
import * as smartrx from '@push.rocks/smartrx';
|
||||
import * as smarturl from '@push.rocks/smarturl';
|
||||
|
||||
export { smartfile, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl };
|
||||
export { smartfile, smartdelay, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl };
|
||||
|
||||
// third party scope
|
||||
import * as fileType from 'file-type';
|
||||
|
Reference in New Issue
Block a user