Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e3c5a8443 | |||
| 11bbddc763 | |||
| 5cc030d433 | |||
| 2b91c1586c | |||
| e609c023bc | |||
| 634c204a00 | |||
| 7ccd210c45 | |||
| fe90de56d6 | |||
| d9251fa1a5 | |||
| ec58b9cdc5 | |||
| 9dbb7d9731 | |||
| 4428638170 | |||
| 1af585594c | |||
| 780db4921e | |||
| ed5f590b5f | |||
| a32ed0facd | |||
| b5a3793ed5 | |||
| be1bc958d8 | |||
| 21434622dd |
@@ -6,8 +6,8 @@ on:
|
|||||||
- '**'
|
- '**'
|
||||||
|
|
||||||
env:
|
env:
|
||||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||||
@@ -26,7 +26,7 @@ jobs:
|
|||||||
- name: Install pnpm and npmci
|
- name: Install pnpm and npmci
|
||||||
run: |
|
run: |
|
||||||
pnpm install -g pnpm
|
pnpm install -g pnpm
|
||||||
pnpm install -g @shipzone/npmci
|
pnpm install -g @ship.zone/npmci
|
||||||
|
|
||||||
- name: Run npm prepare
|
- name: Run npm prepare
|
||||||
run: npmci npm prepare
|
run: npmci npm prepare
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ on:
|
|||||||
- '*'
|
- '*'
|
||||||
|
|
||||||
env:
|
env:
|
||||||
IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
|
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||||
@@ -26,7 +26,7 @@ jobs:
|
|||||||
- name: Prepare
|
- name: Prepare
|
||||||
run: |
|
run: |
|
||||||
pnpm install -g pnpm
|
pnpm install -g pnpm
|
||||||
pnpm install -g @shipzone/npmci
|
pnpm install -g @ship.zone/npmci
|
||||||
npmci npm prepare
|
npmci npm prepare
|
||||||
|
|
||||||
- name: Audit production dependencies
|
- name: Audit production dependencies
|
||||||
@@ -54,7 +54,7 @@ jobs:
|
|||||||
- name: Prepare
|
- name: Prepare
|
||||||
run: |
|
run: |
|
||||||
pnpm install -g pnpm
|
pnpm install -g pnpm
|
||||||
pnpm install -g @shipzone/npmci
|
pnpm install -g @ship.zone/npmci
|
||||||
npmci npm prepare
|
npmci npm prepare
|
||||||
|
|
||||||
- name: Test stable
|
- name: Test stable
|
||||||
@@ -82,7 +82,7 @@ jobs:
|
|||||||
- name: Prepare
|
- name: Prepare
|
||||||
run: |
|
run: |
|
||||||
pnpm install -g pnpm
|
pnpm install -g pnpm
|
||||||
pnpm install -g @shipzone/npmci
|
pnpm install -g @ship.zone/npmci
|
||||||
npmci npm prepare
|
npmci npm prepare
|
||||||
|
|
||||||
- name: Release
|
- name: Release
|
||||||
@@ -104,7 +104,7 @@ jobs:
|
|||||||
- name: Prepare
|
- name: Prepare
|
||||||
run: |
|
run: |
|
||||||
pnpm install -g pnpm
|
pnpm install -g pnpm
|
||||||
pnpm install -g @shipzone/npmci
|
pnpm install -g @ship.zone/npmci
|
||||||
npmci npm prepare
|
npmci npm prepare
|
||||||
|
|
||||||
- name: Code quality
|
- name: Code quality
|
||||||
|
|||||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -3,7 +3,6 @@
|
|||||||
# artifacts
|
# artifacts
|
||||||
coverage/
|
coverage/
|
||||||
public/
|
public/
|
||||||
pages/
|
|
||||||
|
|
||||||
# installs
|
# installs
|
||||||
node_modules/
|
node_modules/
|
||||||
@@ -17,4 +16,8 @@ node_modules/
|
|||||||
dist/
|
dist/
|
||||||
dist_*/
|
dist_*/
|
||||||
|
|
||||||
# custom
|
# AI
|
||||||
|
.claude/
|
||||||
|
.serena/
|
||||||
|
|
||||||
|
#------# custom
|
||||||
97
changelog.md
97
changelog.md
@@ -1,6 +1,97 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2025-11-25 - 5.0.1 - fix(ziptools,gziptools)
|
||||||
|
Use fflate synchronous APIs for ZIP and GZIP operations for Deno compatibility; add TEntryFilter type and small docs/tests cleanup
|
||||||
|
|
||||||
|
- Replace fflate async APIs (zip, unzip, gzip, gunzip with callbacks) with synchronous counterparts (zipSync, unzipSync, gzipSync, gunzipSync) to avoid Web Worker issues in Deno
|
||||||
|
- ZipCompressionStream.finalize now uses fflate.zipSync and emits compressed Buffer synchronously
|
||||||
|
- GzipTools.compress / decompress now delegate to compressSync / decompressSync for cross-runtime compatibility
|
||||||
|
- ZipTools.createZip and ZipTools.extractZip now use zipSync/unzipSync and return Buffers
|
||||||
|
- Add TEntryFilter type to ts/interfaces.ts for fluent API entry filtering
|
||||||
|
- Minor readme.hints.md updates and small whitespace tidy in tests
|
||||||
|
|
||||||
|
## 2025-11-25 - 5.0.0 - BREAKING CHANGE(SmartArchive)
|
||||||
|
Refactor public API: rename factory/extraction methods, introduce typed interfaces and improved compression tools
|
||||||
|
|
||||||
|
- Renamed SmartArchive factory methods: fromArchiveUrl -> fromUrl, fromArchiveFile -> fromFile, fromArchiveStream -> fromStream; added fromBuffer helper.
|
||||||
|
- Renamed extraction APIs: exportToFs -> extractToDirectory and exportToStreamOfStreamFiles -> extractToStream; stream-based helpers updated accordingly.
|
||||||
|
- Export surface reorganized (ts/index.ts): core interfaces and errors are exported and new modules (bzip2tools, archiveanalyzer) are publicly available.
|
||||||
|
- Introduced strong TypeScript types (ts/interfaces.ts) and centralized error types (ts/errors.ts) including Bzip2Error and BZIP2_ERROR_CODES.
|
||||||
|
- Refactored format implementations and stream transforms: GzipTools/GzipCompressionTransform/GzipDecompressionTransform, ZipTools (ZipCompressionStream, ZipDecompressionTransform), TarTools improvements.
|
||||||
|
- BZIP2 implementation improvements: new bit iterator (IBitReader), clearer error handling and streaming unbzip2 transform.
|
||||||
|
- Updated tests to use the new APIs and method names.
|
||||||
|
- Breaking change: public API method names and some class/transform names have changed — this requires code updates for consumers.
|
||||||
|
|
||||||
|
## 2025-11-25 - 4.2.4 - fix(plugins)
|
||||||
|
Migrate filesystem usage to Node fs/fsPromises and upgrade smartfile to v13; add listFileTree helper and update tests
|
||||||
|
|
||||||
|
- Bumped dependency @push.rocks/smartfile to ^13.0.0 and removed unused dependency `through`
|
||||||
|
- Replaced usages of smartfile.fs and smartfile.fsStream with Node native fs and fs/promises (createReadStream/createWriteStream, mkdir({recursive:true}), stat, readFile)
|
||||||
|
- Added plugins.listFileTree helper (recursive directory lister) and used it in TarTools.packDirectory and tests
|
||||||
|
- Updated SmartArchive.exportToFs to use plugins.fs and plugins.fsPromises for directory creation and file writes
|
||||||
|
- Updated TarTools to use plugins.fs.createReadStream and plugins.fsPromises.stat when packing directories
|
||||||
|
- Converted/updated tests to a Node/Deno-friendly test file (test.node+deno.ts) and switched test helpers to use fsPromises
|
||||||
|
- Added readme.hints.md with migration notes for Smartfile v13 and architecture/dependency notes
|
||||||
|
|
||||||
|
## 2025-11-25 - 4.2.3 - fix(build)
|
||||||
|
Upgrade dev tooling: bump @git.zone/tsbuild, @git.zone/tsrun and @git.zone/tstest versions
|
||||||
|
|
||||||
|
- Bump @git.zone/tsbuild from ^2.6.6 to ^3.1.0
|
||||||
|
- Bump @git.zone/tsrun from ^1.3.3 to ^2.0.0
|
||||||
|
- Bump @git.zone/tstest from ^2.3.4 to ^3.1.3
|
||||||
|
|
||||||
|
## 2025-08-18 - 4.2.2 - fix(smartarchive)
|
||||||
|
Improve tar entry streaming handling and add in-memory gzip/tgz tests
|
||||||
|
|
||||||
|
- Fix tar entry handling: properly consume directory entries (resume stream) and wait for entry end before continuing to next header
|
||||||
|
- Wrap tar file entries with a PassThrough so extracted StreamFile instances can be consumed while the tar extractor continues
|
||||||
|
- Handle nested archives correctly by piping resultStream -> decompressionStream -> analyzer -> unpacker, avoiding premature end signals
|
||||||
|
- Add and expand tests in test/test.gzip.ts: verify package.json and TS/license files after extraction, add in-memory gzip extraction test, and add real tgz-in-memory download+extraction test
|
||||||
|
- Minor logging improvements for tar extraction flow
|
||||||
|
|
||||||
|
## 2025-08-18 - 4.2.1 - fix(gzip)
|
||||||
|
Improve gzip streaming decompression, archive analysis and unpacking; add gzip tests
|
||||||
|
|
||||||
|
- Add a streaming DecompressGunzipTransform using fflate.Gunzip with proper _flush handling to support chunked gzip input and avoid buffering issues.
|
||||||
|
- Refactor ArchiveAnalyzer: introduce IAnalyzedResult, getAnalyzedStream(), and getDecompressionStream() to better detect mime types and wire appropriate decompression streams (gzip, zip, bzip2, tar).
|
||||||
|
- Use SmartRequest response streams converted via stream.Readable.fromWeb for URL sources in SmartArchive.getArchiveStream() to improve remote archive handling.
|
||||||
|
- Improve nested archive unpacking and SmartArchive export pipeline: more robust tar/zip handling, consistent SmartDuplex usage and backpressure handling.
|
||||||
|
- Enhance exportToFs: ensure directories, improved logging for relative paths, and safer write-stream wiring.
|
||||||
|
- Add comprehensive gzip-focused tests (test/test.gzip.ts) covering file extraction, stream extraction, header filename handling, large files, and a real-world tgz-from-URL extraction scenario.
|
||||||
|
|
||||||
|
## 2025-08-18 - 4.2.0 - feat(classes.smartarchive)
|
||||||
|
Support URL streams, recursive archive unpacking and filesystem export; improve ZIP/GZIP/BZIP2 robustness; CI and package metadata updates
|
||||||
|
|
||||||
|
- Add exportToFs(targetDir, fileName?) to write extracted StreamFile objects to the filesystem (ensures directories, logs relative paths, waits for write completion).
|
||||||
|
- Implement exportToStreamOfStreamFiles with recursive unpacking pipeline that handles application/x-tar (tar-stream Extract), application/zip (fflate Unzip), nested archives and StreamIntake for StreamFile results.
|
||||||
|
- Enhance getArchiveStream() to support URL/web streams (SmartRequest) and return Node Readable streams for remote archives.
|
||||||
|
- Make ZIP decompression more robust: accept ArrayBuffer-like chunks, coerce to Buffer before pushing to fflate.Unzip, and ensure SmartDuplex handling of results.
|
||||||
|
- Fixes and improvements to bzip2/gzip/tar tool implementations (various bug/formatting fixes, safer CRC and stream handling).
|
||||||
|
- Update CI workflows to use new registry image and adjust npmci install path; minor .gitignore additions.
|
||||||
|
- Package metadata tweaks: bugs URL and homepage updated, packageManager/pnpm fields adjusted.
|
||||||
|
- Documentation/readme expanded and polished with quick start, examples and API reference updates.
|
||||||
|
- Small test and plugin export cleanups (formatting and trailing commas removed/added).
|
||||||
|
- TypeScript/formatting fixes across many files (consistent casing, trailing commas, typings, tsconfig additions).
|
||||||
|
|
||||||
|
## 2025-08-18 - 4.1.0 - feat(classes.smartarchive)
|
||||||
|
|
||||||
|
Support URL web streams, add recursive archive unpacking and filesystem export, and improve ZIP decompression robustness
|
||||||
|
|
||||||
|
- ts/classes.smartarchive.ts: add exportToFs(targetDir, fileName?) to write extracted StreamFile objects to the filesystem (ensures directories, logs relative paths, waits for write completion).
|
||||||
|
- ts/classes.smartarchive.ts: implement exportToStreamOfStreamFiles with recursive unpacking pipeline that handles application/x-tar (tar-stream Extract), application/zip (fflate unzip), nested archives and StreamIntake for StreamFile results.
|
||||||
|
- ts/classes.smartarchive.ts: improve getArchiveStream() for URL sources by using SmartRequest.create().url(...).get() and converting the returned Web stream into a Node Readable stream.
|
||||||
|
- ts/classes.ziptools.ts: make ZIP decompression writeFunction more robust — accept non-Buffer chunks, coerce to Buffer before pushing to fflate.Unzip, and loosen the writeFunction typing to handle incoming ArrayBuffer-like data.
|
||||||
|
|
||||||
|
## 2024-10-13 - 4.0.39 - fix(core)
|
||||||
|
|
||||||
|
Fix dependencies and update documentation.
|
||||||
|
|
||||||
|
- Ensure package uses the latest dependencies
|
||||||
|
- Reviewed and grouped imports in TypeScript files
|
||||||
|
- Updated readme with advanced usage examples
|
||||||
|
|
||||||
## 2024-10-13 - 4.0.38 - fix(dependencies)
|
## 2024-10-13 - 4.0.38 - fix(dependencies)
|
||||||
|
|
||||||
Update dependencies to latest versions
|
Update dependencies to latest versions
|
||||||
|
|
||||||
- Updated @push.rocks/smartfile to version 11.0.21
|
- Updated @push.rocks/smartfile to version 11.0.21
|
||||||
@@ -13,11 +104,13 @@ Update dependencies to latest versions
|
|||||||
- Updated @push.rocks/tapbundle to version 5.3.0
|
- Updated @push.rocks/tapbundle to version 5.3.0
|
||||||
|
|
||||||
## 2024-06-08 - 4.0.24 to 4.0.37 - Fixes and Updates
|
## 2024-06-08 - 4.0.24 to 4.0.37 - Fixes and Updates
|
||||||
|
|
||||||
Core updates and bug fixes were implemented in versions 4.0.24 through 4.0.37.
|
Core updates and bug fixes were implemented in versions 4.0.24 through 4.0.37.
|
||||||
|
|
||||||
- Repeated core updates and fixes applied consistently across multiple versions.
|
- Repeated core updates and fixes applied consistently across multiple versions.
|
||||||
|
|
||||||
## 2024-06-06 - 4.0.22 to 4.0.23 - Descriptions and Fixes Updates
|
## 2024-06-06 - 4.0.22 to 4.0.23 - Descriptions and Fixes Updates
|
||||||
|
|
||||||
Efforts to update documentation and core features.
|
Efforts to update documentation and core features.
|
||||||
|
|
||||||
- "update description" in 4.0.22
|
- "update description" in 4.0.22
|
||||||
@@ -25,22 +118,26 @@ Efforts to update documentation and core features.
|
|||||||
- Ongoing core fixes.
|
- Ongoing core fixes.
|
||||||
|
|
||||||
## 2023-11-06 - 4.0.0 - Major Update with Breaking Changes
|
## 2023-11-06 - 4.0.0 - Major Update with Breaking Changes
|
||||||
|
|
||||||
Introduction of significant updates and breaking changes.
|
Introduction of significant updates and breaking changes.
|
||||||
|
|
||||||
- Transition to new version 4.0.0 with core updates.
|
- Transition to new version 4.0.0 with core updates.
|
||||||
- Break in compatibility due to major structural changes with core functionalities.
|
- Break in compatibility due to major structural changes with core functionalities.
|
||||||
|
|
||||||
## 2023-07-11 - 3.0.6 - Organizational Changes
|
## 2023-07-11 - 3.0.6 - Organizational Changes
|
||||||
|
|
||||||
Structural reorganization and updates to the organization schema.
|
Structural reorganization and updates to the organization schema.
|
||||||
|
|
||||||
- Switch to new organizational schema implemented.
|
- Switch to new organizational schema implemented.
|
||||||
|
|
||||||
## 2022-04-04 - 3.0.0 - Build Updates and Breaking Changes
|
## 2022-04-04 - 3.0.0 - Build Updates and Breaking Changes
|
||||||
|
|
||||||
Major build update introducing breaking changes.
|
Major build update introducing breaking changes.
|
||||||
|
|
||||||
- Introduction of ESM structure with breaking changes.
|
- Introduction of ESM structure with breaking changes.
|
||||||
|
|
||||||
## 2016-01-18 - 0.0.0 to 1.0.0 - Initial Development and Launch
|
## 2016-01-18 - 0.0.0 to 1.0.0 - Initial Development and Launch
|
||||||
|
|
||||||
Initial software development and establishment of core features.
|
Initial software development and establishment of core features.
|
||||||
|
|
||||||
- Project set-up including Travis CI integration.
|
- Project set-up including Travis CI integration.
|
||||||
|
|||||||
4
dist_ts/index.d.ts
vendored
4
dist_ts/index.d.ts
vendored
@@ -1,4 +1,8 @@
|
|||||||
|
export * from './interfaces.js';
|
||||||
|
export * from './errors.js';
|
||||||
export * from './classes.smartarchive.js';
|
export * from './classes.smartarchive.js';
|
||||||
export * from './classes.tartools.js';
|
export * from './classes.tartools.js';
|
||||||
export * from './classes.ziptools.js';
|
export * from './classes.ziptools.js';
|
||||||
export * from './classes.gziptools.js';
|
export * from './classes.gziptools.js';
|
||||||
|
export * from './classes.bzip2tools.js';
|
||||||
|
export * from './classes.archiveanalyzer.js';
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
|
// Core types and errors
|
||||||
|
export * from './interfaces.js';
|
||||||
|
export * from './errors.js';
|
||||||
|
// Main archive class
|
||||||
export * from './classes.smartarchive.js';
|
export * from './classes.smartarchive.js';
|
||||||
|
// Format-specific tools
|
||||||
export * from './classes.tartools.js';
|
export * from './classes.tartools.js';
|
||||||
export * from './classes.ziptools.js';
|
export * from './classes.ziptools.js';
|
||||||
export * from './classes.gziptools.js';
|
export * from './classes.gziptools.js';
|
||||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxjQUFjLDJCQUEyQixDQUFDO0FBQzFDLGNBQWMsdUJBQXVCLENBQUM7QUFDdEMsY0FBYyx1QkFBdUIsQ0FBQztBQUN0QyxjQUFjLHdCQUF3QixDQUFDIn0=
|
export * from './classes.bzip2tools.js';
|
||||||
|
// Archive analysis
|
||||||
|
export * from './classes.archiveanalyzer.js';
|
||||||
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSx3QkFBd0I7QUFDeEIsY0FBYyxpQkFBaUIsQ0FBQztBQUNoQyxjQUFjLGFBQWEsQ0FBQztBQUU1QixxQkFBcUI7QUFDckIsY0FBYywyQkFBMkIsQ0FBQztBQUUxQyx3QkFBd0I7QUFDeEIsY0FBYyx1QkFBdUIsQ0FBQztBQUN0QyxjQUFjLHVCQUF1QixDQUFDO0FBQ3RDLGNBQWMsd0JBQXdCLENBQUM7QUFDdkMsY0FBYyx5QkFBeUIsQ0FBQztBQUV4QyxtQkFBbUI7QUFDbkIsY0FBYyw4QkFBOEIsQ0FBQyJ9
|
||||||
40
package.json
40
package.json
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@push.rocks/smartarchive",
|
"name": "@push.rocks/smartarchive",
|
||||||
"version": "4.0.38",
|
"version": "5.0.1",
|
||||||
"description": "A library for working with archive files, providing utilities for compressing and decompressing data.",
|
"description": "A library for working with archive files, providing utilities for compressing and decompressing data.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
"typings": "dist_ts/index.d.ts",
|
"typings": "dist_ts/index.d.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "(tstest test/ --web)",
|
"test": "(tstest test/ --verbose)",
|
||||||
"build": "tsbuild --web --allowimplicitany",
|
"build": "tsbuild --web --allowimplicitany",
|
||||||
"buildDocs": "tsdoc"
|
"buildDocs": "tsdoc"
|
||||||
},
|
},
|
||||||
@@ -17,30 +17,28 @@
|
|||||||
"author": "Lossless GmbH",
|
"author": "Lossless GmbH",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"bugs": {
|
"bugs": {
|
||||||
"url": "https://github.com/pushrocks/smartarchive/issues"
|
"url": "https://code.foss.global/push.rocks/smartarchive/issues"
|
||||||
},
|
},
|
||||||
"homepage": "https://code.foss.global/push.rocks/smartarchive",
|
"homepage": "https://code.foss.global/push.rocks/smartarchive#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@push.rocks/smartdelay": "^3.0.5",
|
"@push.rocks/smartdelay": "^3.0.5",
|
||||||
"@push.rocks/smartfile": "^11.0.21",
|
"@push.rocks/smartfile": "^13.0.0",
|
||||||
"@push.rocks/smartpath": "^5.0.18",
|
"@push.rocks/smartpath": "^6.0.0",
|
||||||
"@push.rocks/smartpromise": "^4.0.4",
|
"@push.rocks/smartpromise": "^4.2.3",
|
||||||
"@push.rocks/smartrequest": "^2.0.22",
|
"@push.rocks/smartrequest": "^4.2.2",
|
||||||
"@push.rocks/smartrx": "^3.0.7",
|
"@push.rocks/smartrx": "^3.0.10",
|
||||||
"@push.rocks/smartstream": "^3.0.46",
|
"@push.rocks/smartstream": "^3.2.5",
|
||||||
"@push.rocks/smartunique": "^3.0.9",
|
"@push.rocks/smartunique": "^3.0.9",
|
||||||
"@push.rocks/smarturl": "^3.1.0",
|
"@push.rocks/smarturl": "^3.1.0",
|
||||||
"@types/tar-stream": "^3.1.3",
|
"@types/tar-stream": "^3.1.4",
|
||||||
"fflate": "^0.8.2",
|
"fflate": "^0.8.2",
|
||||||
"file-type": "^19.5.0",
|
"file-type": "^21.0.0",
|
||||||
"tar-stream": "^3.1.7",
|
"tar-stream": "^3.1.7"
|
||||||
"through": "^2.3.8"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@git.zone/tsbuild": "^2.1.84",
|
"@git.zone/tsbuild": "^3.1.0",
|
||||||
"@git.zone/tsrun": "^1.2.49",
|
"@git.zone/tsrun": "^2.0.0",
|
||||||
"@git.zone/tstest": "^1.0.90",
|
"@git.zone/tstest": "^3.1.3"
|
||||||
"@push.rocks/tapbundle": "^5.3.0"
|
|
||||||
},
|
},
|
||||||
"private": false,
|
"private": false,
|
||||||
"files": [
|
"files": [
|
||||||
@@ -70,5 +68,9 @@
|
|||||||
"file creation",
|
"file creation",
|
||||||
"data analysis",
|
"data analysis",
|
||||||
"file stream"
|
"file stream"
|
||||||
]
|
],
|
||||||
|
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748",
|
||||||
|
"pnpm": {
|
||||||
|
"overrides": {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
9412
pnpm-lock.yaml
generated
9412
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
4
pnpm-workspace.yaml
Normal file
4
pnpm-workspace.yaml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
onlyBuiltDependencies:
|
||||||
|
- esbuild
|
||||||
|
- mongodb-memory-server
|
||||||
|
- puppeteer
|
||||||
@@ -1 +1,84 @@
|
|||||||
|
# Smartarchive Development Hints
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
`@push.rocks/smartarchive` uses a **fluent builder pattern** for all archive operations. The main entry point is `SmartArchive.create()` which returns a builder instance.
|
||||||
|
|
||||||
|
### Two Operating Modes
|
||||||
|
|
||||||
|
1. **Extraction Mode** - Triggered by `.url()`, `.file()`, `.stream()`, or `.buffer()`
|
||||||
|
2. **Creation Mode** - Triggered by `.format()` or `.entry()`
|
||||||
|
|
||||||
|
Modes are mutually exclusive - you cannot mix extraction and creation methods in the same chain.
|
||||||
|
|
||||||
|
## Key Classes
|
||||||
|
|
||||||
|
- **SmartArchive** - Main class with fluent API for all operations
|
||||||
|
- **TarTools** - TAR-specific operations (pack/extract)
|
||||||
|
- **ZipTools** - ZIP-specific operations using fflate
|
||||||
|
- **GzipTools** - GZIP compression/decompression using fflate
|
||||||
|
- **Bzip2Tools** - BZIP2 decompression (extract only, no creation)
|
||||||
|
- **ArchiveAnalyzer** - Format detection via magic bytes
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- **fflate** - Pure JS compression for ZIP/GZIP (works in browser)
|
||||||
|
- **tar-stream** - TAR archive handling
|
||||||
|
- **file-type** - MIME type detection via magic bytes
|
||||||
|
- **@push.rocks/smartfile** - SmartFile and StreamFile classes
|
||||||
|
|
||||||
|
## API Changes (v5.0.0)
|
||||||
|
|
||||||
|
The v5.0.0 release introduced a complete API refactor:
|
||||||
|
|
||||||
|
### Old API (deprecated)
|
||||||
|
```typescript
|
||||||
|
// Old static factory methods - NO LONGER EXIST
|
||||||
|
await SmartArchive.fromUrl(url);
|
||||||
|
await SmartArchive.fromFile(path);
|
||||||
|
await SmartArchive.fromDirectory(path, options);
|
||||||
|
```
|
||||||
|
|
||||||
|
### New Fluent API
|
||||||
|
```typescript
|
||||||
|
// Current fluent builder pattern
|
||||||
|
await SmartArchive.create()
|
||||||
|
.url(url)
|
||||||
|
.extract(targetDir);
|
||||||
|
|
||||||
|
await SmartArchive.create()
|
||||||
|
.format('tar.gz')
|
||||||
|
.directory(path)
|
||||||
|
.toFile(outputPath);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Migration Notes (from v4.x)
|
||||||
|
|
||||||
|
### Smartfile v13 Changes
|
||||||
|
Smartfile v13 removed filesystem operations. Replacements:
|
||||||
|
- `smartfile.fs.ensureDir(path)` → `fsPromises.mkdir(path, { recursive: true })`
|
||||||
|
- `smartfile.fs.stat(path)` → `fsPromises.stat(path)`
|
||||||
|
- `smartfile.fs.toReadStream(path)` → `fs.createReadStream(path)`
|
||||||
|
|
||||||
|
### Still using from smartfile
|
||||||
|
- `SmartFile` class (in-memory file representation)
|
||||||
|
- `StreamFile` class (streaming file handling)
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pnpm test # Run all tests
|
||||||
|
tstest test/test.node+deno.ts --verbose # Run specific test
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests use a Verdaccio registry URL (`verdaccio.lossless.digital`) for test archives.
|
||||||
|
|
||||||
|
## Key Files
|
||||||
|
|
||||||
|
- `ts/classes.smartarchive.ts` - Main SmartArchive class with fluent API
|
||||||
|
- `ts/classes.tartools.ts` - TAR operations
|
||||||
|
- `ts/classes.ziptools.ts` - ZIP operations
|
||||||
|
- `ts/classes.gziptools.ts` - GZIP operations
|
||||||
|
- `ts/classes.bzip2tools.ts` - BZIP2 decompression
|
||||||
|
- `ts/classes.archiveanalyzer.ts` - Format detection
|
||||||
|
- `ts/interfaces.ts` - Type definitions
|
||||||
|
|||||||
602
readme.md
602
readme.md
@@ -1,266 +1,526 @@
|
|||||||
# @push.rocks/smartarchive
|
# @push.rocks/smartarchive 📦
|
||||||
|
|
||||||
`@push.rocks/smartarchive` is a powerful library designed for managing archive files. It provides utilities for compressing and decompressing data in various formats such as zip, tar, gzip, and bzip2. This library aims to simplify the process of handling archive files, making it an ideal choice for projects that require manipulation of archived data.
|
A powerful, streaming-first archive manipulation library with a fluent builder API. Works seamlessly in Node.js and Deno.
|
||||||
|
|
||||||
## Install
|
## Issue Reporting and Security
|
||||||
|
|
||||||
To install `@push.rocks/smartarchive`, you can either use npm or yarn. Run one of the following commands in your project directory:
|
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||||
|
|
||||||
```shell
|
## Features 🚀
|
||||||
npm install @push.rocks/smartarchive --save
|
|
||||||
```
|
|
||||||
|
|
||||||
or if you prefer yarn:
|
- 📁 **Multi-format support** – Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, `.gz`, and `.bz2` archives
|
||||||
|
- 🌊 **Streaming-first architecture** – Process large archives without memory constraints
|
||||||
|
- ✨ **Fluent builder API** – Chain methods for readable, expressive code
|
||||||
|
- 🎯 **Smart detection** – Automatically identifies archive types via magic bytes
|
||||||
|
- ⚡ **High performance** – Built on `tar-stream` and `fflate` for speed
|
||||||
|
- 🔧 **Flexible I/O** – Work with files, URLs, streams, and buffers seamlessly
|
||||||
|
- 🛠️ **Modern TypeScript** – Full type safety and excellent IDE support
|
||||||
|
- 🔄 **Dual-mode operation** – Extract existing archives OR create new ones
|
||||||
|
- 🦕 **Cross-runtime** – Works in both Node.js and Deno environments
|
||||||
|
|
||||||
```shell
|
## Installation 📥
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Using pnpm (recommended)
|
||||||
|
pnpm add @push.rocks/smartarchive
|
||||||
|
|
||||||
|
# Using npm
|
||||||
|
npm install @push.rocks/smartarchive
|
||||||
|
|
||||||
|
# Using yarn
|
||||||
yarn add @push.rocks/smartarchive
|
yarn add @push.rocks/smartarchive
|
||||||
```
|
```
|
||||||
|
|
||||||
This will add `@push.rocks/smartarchive` to your project's dependencies.
|
## Quick Start 🎯
|
||||||
|
|
||||||
## Usage
|
### Extract an archive from URL
|
||||||
`@push.rocks/smartarchive` provides an easy-to-use API for extracting, creating, and analyzing archive files. Below, we'll cover how to get started and explore various features of the module.
|
|
||||||
|
|
||||||
### Importing SmartArchive
|
|
||||||
|
|
||||||
First, import `SmartArchive` from `@push.rocks/smartarchive` using ESM syntax:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
// Extract a .tar.gz archive from a URL directly to the filesystem
|
||||||
|
await SmartArchive.create()
|
||||||
|
.url('https://registry.npmjs.org/some-package/-/some-package-1.0.0.tgz')
|
||||||
|
.extract('./extracted');
|
||||||
```
|
```
|
||||||
|
|
||||||
### Extracting Archive Files
|
### Create an archive from entries
|
||||||
|
|
||||||
You can extract archive files from different sources using `SmartArchive.fromArchiveUrl`, `SmartArchive.fromArchiveFile`, and `SmartArchive.fromArchiveStream`. Here's an example of extracting an archive from a URL:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function extractArchiveFromURL() {
|
// Create a tar.gz archive with files
|
||||||
const url = 'https://example.com/archive.zip';
|
await SmartArchive.create()
|
||||||
const targetDir = '/path/to/extract';
|
.format('tar.gz')
|
||||||
|
.compression(6)
|
||||||
|
.entry('config.json', JSON.stringify({ name: 'myapp' }))
|
||||||
|
.entry('readme.txt', 'Hello World!')
|
||||||
|
.toFile('./backup.tar.gz');
|
||||||
|
```
|
||||||
|
|
||||||
const archive = await SmartArchive.fromArchiveUrl(url);
|
### Extract with filtering and path manipulation
|
||||||
await archive.exportToFs(targetDir);
|
|
||||||
|
|
||||||
console.log('Archive extracted successfully.');
|
```typescript
|
||||||
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
// Extract only JSON files, stripping the first path component
|
||||||
|
await SmartArchive.create()
|
||||||
|
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||||
|
.stripComponents(1) // Remove 'package/' prefix
|
||||||
|
.include(/\.json$/) // Only extract JSON files
|
||||||
|
.extract('./node_modules/lodash');
|
||||||
|
```
|
||||||
|
|
||||||
|
## Core Concepts 💡
|
||||||
|
|
||||||
|
### Fluent Builder Pattern
|
||||||
|
|
||||||
|
`SmartArchive` uses a fluent builder pattern where you chain methods to configure the operation:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
SmartArchive.create() // Start a new builder
|
||||||
|
.source(...) // Configure source (extraction mode)
|
||||||
|
.options(...) // Set options
|
||||||
|
.terminal() // Execute the operation
|
||||||
|
```
|
||||||
|
|
||||||
|
### Two Operating Modes
|
||||||
|
|
||||||
|
**Extraction Mode** - Load an existing archive and extract/analyze it:
|
||||||
|
```typescript
|
||||||
|
SmartArchive.create()
|
||||||
|
.url('...') // or .file(), .stream(), .buffer()
|
||||||
|
.extract('./out') // or .toSmartFiles(), .list(), etc.
|
||||||
|
```
|
||||||
|
|
||||||
|
**Creation Mode** - Build a new archive from entries:
|
||||||
|
```typescript
|
||||||
|
SmartArchive.create()
|
||||||
|
.format('tar.gz') // Set output format
|
||||||
|
.entry(...) // Add files
|
||||||
|
.toFile('./out.tar.gz') // or .toBuffer(), .toStream()
|
||||||
|
```
|
||||||
|
|
||||||
|
> ⚠️ **Note:** You cannot mix extraction and creation methods in the same chain.
|
||||||
|
|
||||||
|
## API Reference 📚
|
||||||
|
|
||||||
|
### Source Methods (Extraction Mode)
|
||||||
|
|
||||||
|
| Method | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `.url(url)` | Load archive from a URL |
|
||||||
|
| `.file(path)` | Load archive from local filesystem |
|
||||||
|
| `.stream(readable)` | Load archive from any Node.js readable stream |
|
||||||
|
| `.buffer(buffer)` | Load archive from an in-memory Buffer |
|
||||||
|
|
||||||
|
### Creation Methods (Creation Mode)
|
||||||
|
|
||||||
|
| Method | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `.format(fmt)` | Set output format: `'tar'`, `'tar.gz'`, `'tgz'`, `'zip'`, `'gz'` |
|
||||||
|
| `.compression(level)` | Set compression level (0-9, default: 6) |
|
||||||
|
| `.entry(path, content)` | Add a file entry (string or Buffer content) |
|
||||||
|
| `.entries(array)` | Add multiple entries at once |
|
||||||
|
| `.directory(path, archiveBase?)` | Add entire directory contents |
|
||||||
|
| `.addSmartFile(file, path?)` | Add a SmartFile instance |
|
||||||
|
| `.addStreamFile(file, path?)` | Add a StreamFile instance |
|
||||||
|
|
||||||
|
### Filter Methods (Both Modes)
|
||||||
|
|
||||||
|
| Method | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `.filter(predicate)` | Filter entries with custom function |
|
||||||
|
| `.include(pattern)` | Only include entries matching regex/string pattern |
|
||||||
|
| `.exclude(pattern)` | Exclude entries matching regex/string pattern |
|
||||||
|
|
||||||
|
### Extraction Options
|
||||||
|
|
||||||
|
| Method | Description |
|
||||||
|
|--------|-------------|
|
||||||
|
| `.stripComponents(n)` | Strip N leading path components |
|
||||||
|
| `.overwrite(bool)` | Overwrite existing files (default: false) |
|
||||||
|
| `.fileName(name)` | Set output filename for single-file archives (gz, bz2) |
|
||||||
|
|
||||||
|
### Terminal Methods (Extraction)
|
||||||
|
|
||||||
|
| Method | Returns | Description |
|
||||||
|
|--------|---------|-------------|
|
||||||
|
| `.extract(targetDir)` | `Promise<void>` | Extract to filesystem directory |
|
||||||
|
| `.toStreamFiles()` | `Promise<StreamIntake<StreamFile>>` | Get stream of StreamFile objects |
|
||||||
|
| `.toSmartFiles()` | `Promise<SmartFile[]>` | Get in-memory SmartFile array |
|
||||||
|
| `.extractFile(path)` | `Promise<SmartFile \| null>` | Extract single file by path |
|
||||||
|
| `.list()` | `Promise<IArchiveEntryInfo[]>` | List all entries |
|
||||||
|
| `.analyze()` | `Promise<IArchiveInfo>` | Get archive metadata |
|
||||||
|
| `.hasFile(path)` | `Promise<boolean>` | Check if file exists |
|
||||||
|
|
||||||
|
### Terminal Methods (Creation)
|
||||||
|
|
||||||
|
| Method | Returns | Description |
|
||||||
|
|--------|---------|-------------|
|
||||||
|
| `.build()` | `Promise<SmartArchive>` | Build the archive (implicit in other terminals) |
|
||||||
|
| `.toBuffer()` | `Promise<Buffer>` | Get archive as Buffer |
|
||||||
|
| `.toFile(path)` | `Promise<void>` | Write archive to disk |
|
||||||
|
| `.toStream()` | `Promise<Readable>` | Get raw archive stream |
|
||||||
|
|
||||||
|
## Usage Examples 🔨
|
||||||
|
|
||||||
|
### Download and extract npm packages
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
const pkg = await SmartArchive.create()
|
||||||
|
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz');
|
||||||
|
|
||||||
|
// Quick inspection of package.json
|
||||||
|
const pkgJson = await pkg.extractFile('package/package.json');
|
||||||
|
if (pkgJson) {
|
||||||
|
const metadata = JSON.parse(pkgJson.contents.toString());
|
||||||
|
console.log(`Package: ${metadata.name}@${metadata.version}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
extractArchiveFromURL();
|
// Full extraction with path normalization
|
||||||
|
await SmartArchive.create()
|
||||||
|
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||||
|
.stripComponents(1)
|
||||||
|
.extract('./node_modules/lodash');
|
||||||
```
|
```
|
||||||
|
|
||||||
### Extracting an Archive from a File
|
### Create ZIP archive
|
||||||
|
|
||||||
Similarly, you can extract an archive from a local file:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function extractArchiveFromFile() {
|
await SmartArchive.create()
|
||||||
const filePath = '/path/to/archive.zip';
|
.format('zip')
|
||||||
const targetDir = '/path/to/extract';
|
.compression(9)
|
||||||
|
.entry('report.txt', 'Monthly sales report...')
|
||||||
const archive = await SmartArchive.fromArchiveFile(filePath);
|
.entry('data/figures.json', JSON.stringify({ revenue: 10000 }))
|
||||||
await archive.exportToFs(targetDir);
|
.entry('images/logo.png', pngBuffer)
|
||||||
|
.toFile('./report-bundle.zip');
|
||||||
console.log('Archive extracted successfully.');
|
|
||||||
}
|
|
||||||
|
|
||||||
extractArchiveFromFile();
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Stream-Based Extraction
|
### Create TAR.GZ from directory
|
||||||
|
|
||||||
For larger files, you might prefer a streaming approach to prevent high memory consumption. Here’s an example:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
|
||||||
import { createReadStream } from 'fs';
|
|
||||||
|
|
||||||
async function extractArchiveUsingStream() {
|
|
||||||
const archiveStream = createReadStream('/path/to/archive.zip');
|
|
||||||
const archive = await SmartArchive.fromArchiveStream(archiveStream);
|
|
||||||
const extractionStream = await archive.exportToStreamOfStreamFiles();
|
|
||||||
|
|
||||||
extractionStream.pipe(createWriteStream('/path/to/destination'));
|
|
||||||
}
|
|
||||||
|
|
||||||
extractArchiveUsingStream();
|
|
||||||
```
|
|
||||||
|
|
||||||
### Analyzing Archive Files
|
|
||||||
|
|
||||||
Sometimes, you may need to inspect the contents of an archive before extracting it. The following example shows how to analyze an archive:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function analyzeArchive() {
|
await SmartArchive.create()
|
||||||
const filePath = '/path/to/archive.zip';
|
.format('tar.gz')
|
||||||
|
.compression(9)
|
||||||
const archive = await SmartArchive.fromArchiveFile(filePath);
|
.directory('./src', 'source') // Archive ./src as 'source/' in archive
|
||||||
const analysisResult = await archive.analyzeContent();
|
.toFile('./project-backup.tar.gz');
|
||||||
|
|
||||||
console.log(analysisResult); // Outputs details about the archive content
|
|
||||||
}
|
|
||||||
|
|
||||||
analyzeArchive();
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Creating Archive Files
|
### Stream-based extraction
|
||||||
|
|
||||||
Creating an archive file is straightforward. Here we demonstrate creating a tar.gz archive:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function createTarGzArchive() {
|
const fileStream = await SmartArchive.create()
|
||||||
const archive = new SmartArchive();
|
.file('./large-archive.tar.gz')
|
||||||
|
.toStreamFiles();
|
||||||
|
|
||||||
// Add directories and files
|
fileStream.on('data', async (streamFile) => {
|
||||||
archive.addedDirectories.push('/path/to/directory1');
|
console.log(`Processing: ${streamFile.relativeFilePath}`);
|
||||||
archive.addedFiles.push('/path/to/file1.txt');
|
|
||||||
|
|
||||||
// Export as tar.gz
|
if (streamFile.relativeFilePath.endsWith('.json')) {
|
||||||
const tarGzStream = await archive.exportToTarGzStream();
|
const content = await streamFile.getContentAsBuffer();
|
||||||
|
const data = JSON.parse(content.toString());
|
||||||
|
// Process JSON data...
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Save to filesystem or handle as needed
|
fileStream.on('end', () => {
|
||||||
tarGzStream.pipe(createWriteStream('/path/to/destination.tar.gz'));
|
console.log('Extraction complete');
|
||||||
}
|
});
|
||||||
|
|
||||||
createTarGzArchive();
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Stream Operations
|
### Filter specific file types
|
||||||
|
|
||||||
Here's an example of using `smartarchive`'s streaming capabilities:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { createReadStream, createWriteStream } from 'fs';
|
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function extractArchiveUsingStreams() {
|
// Extract only TypeScript files
|
||||||
const archiveStream = createReadStream('/path/to/archive.zip');
|
const tsFiles = await SmartArchive.create()
|
||||||
const archive = await SmartArchive.fromArchiveStream(archiveStream);
|
.url('https://example.com/project.tar.gz')
|
||||||
const extractionStream = await archive.exportToStreamOfStreamFiles();
|
.include(/\.ts$/)
|
||||||
|
.exclude(/node_modules/)
|
||||||
|
.toSmartFiles();
|
||||||
|
|
||||||
extractionStream.pipe(createWriteStream('/path/to/extracted'));
|
for (const file of tsFiles) {
|
||||||
|
console.log(`${file.relative}: ${file.contents.length} bytes`);
|
||||||
}
|
}
|
||||||
|
|
||||||
extractArchiveUsingStreams();
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Advanced Decompression Usage
|
### Analyze archive without extraction
|
||||||
|
|
||||||
`smartarchive` supports multiple compression formats. It also provides detailed control over the decompression processes:
|
|
||||||
|
|
||||||
- For ZIP files, `ZipTools` handles decompression using the `fflate` library.
|
|
||||||
- For TAR files, `TarTools` uses `tar-stream`.
|
|
||||||
- For GZIP files, `GzipTools` provides a `CompressGunzipTransform` and `DecompressGunzipTransform`.
|
|
||||||
- For BZIP2 files, `Bzip2Tools` utilizes custom streaming decompression.
|
|
||||||
|
|
||||||
Example: Working with a GZIP-compressed archive:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { createReadStream, createWriteStream } from 'fs';
|
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function decompressGzipArchive() {
|
const archive = SmartArchive.create()
|
||||||
const filePath = '/path/to/archive.gz';
|
.file('./unknown-archive.tar.gz');
|
||||||
const targetDir = '/path/to/extract';
|
|
||||||
|
|
||||||
const archive = await SmartArchive.fromArchiveFile(filePath);
|
// Get format info
|
||||||
await archive.exportToFs(targetDir);
|
const info = await archive.analyze();
|
||||||
|
console.log(`Format: ${info.format}`);
|
||||||
|
console.log(`Compressed: ${info.isCompressed}`);
|
||||||
|
|
||||||
console.log('GZIP archive decompressed successfully.');
|
// List contents
|
||||||
|
const entries = await archive.list();
|
||||||
|
for (const entry of entries) {
|
||||||
|
console.log(`${entry.path} (${entry.isDirectory ? 'dir' : 'file'})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
decompressGzipArchive();
|
// Check for specific file
|
||||||
|
if (await archive.hasFile('package.json')) {
|
||||||
|
const pkgFile = await archive.extractFile('package.json');
|
||||||
|
console.log(pkgFile?.contents.toString());
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Advancing with Custom Decompression Streams
|
### Working with GZIP files
|
||||||
|
|
||||||
You can inject custom decompression streams where needed:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { createReadStream, createWriteStream } from 'fs';
|
|
||||||
import { SmartArchive, GzipTools } from '@push.rocks/smartarchive';
|
import { SmartArchive, GzipTools } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function customDecompression() {
|
// Decompress a .gz file
|
||||||
const filePath = '/path/to/archive.gz';
|
await SmartArchive.create()
|
||||||
const targetDir = '/path/to/extract';
|
.file('./data.json.gz')
|
||||||
|
.fileName('data.json') // Specify output name (gzip doesn't store filename)
|
||||||
|
.extract('./decompressed');
|
||||||
|
|
||||||
const archive = await SmartArchive.fromArchiveFile(filePath);
|
// Use GzipTools directly for compression/decompression
|
||||||
const gzipTools = new GzipTools();
|
const gzipTools = new GzipTools();
|
||||||
const decompressionStream = gzipTools.getDecompressionStream();
|
|
||||||
|
|
||||||
const archiveStream = await archive.getArchiveStream();
|
// Compress a buffer
|
||||||
archiveStream.pipe(decompressionStream).pipe(createWriteStream(targetDir));
|
const compressed = await gzipTools.compress(Buffer.from('Hello World'), 9);
|
||||||
|
const decompressed = await gzipTools.decompress(compressed);
|
||||||
|
|
||||||
console.log('Custom GZIP decompression successful.');
|
// Synchronous operations
|
||||||
}
|
const compressedSync = gzipTools.compressSync(inputBuffer, 6);
|
||||||
|
const decompressedSync = gzipTools.decompressSync(compressedSync);
|
||||||
|
|
||||||
customDecompression();
|
// Streaming
|
||||||
|
const compressStream = gzipTools.getCompressionStream(6);
|
||||||
|
const decompressStream = gzipTools.getDecompressionStream();
|
||||||
|
|
||||||
|
createReadStream('./input.txt')
|
||||||
|
.pipe(compressStream)
|
||||||
|
.pipe(createWriteStream('./output.gz'));
|
||||||
```
|
```
|
||||||
|
|
||||||
### Custom Pack and Unpack Tar
|
### Working with TAR archives directly
|
||||||
|
|
||||||
When dealing with tar archives, you may need to perform custom packing and unpacking:
|
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartArchive, TarTools } from '@push.rocks/smartarchive';
|
import { TarTools } from '@push.rocks/smartarchive';
|
||||||
import { createWriteStream } from 'fs';
|
|
||||||
|
|
||||||
async function customTarOperations() {
|
const tarTools = new TarTools();
|
||||||
const tarTools = new TarTools();
|
|
||||||
|
|
||||||
// Packing a directory into a tar stream
|
// Create a TAR archive manually
|
||||||
const packStream = await tarTools.packDirectory('/path/to/directory');
|
const pack = await tarTools.getPackStream();
|
||||||
packStream.pipe(createWriteStream('/path/to/archive.tar'));
|
|
||||||
|
|
||||||
// Extracting files from a tar stream
|
await tarTools.addFileToPack(pack, {
|
||||||
const extractStream = tarTools.getDecompressionStream();
|
fileName: 'hello.txt',
|
||||||
createReadStream('/path/to/archive.tar').pipe(extractStream).on('entry', (header, stream, next) => {
|
content: 'Hello, World!'
|
||||||
const writeStream = createWriteStream(`/path/to/extract/${header.name}`);
|
});
|
||||||
stream.pipe(writeStream);
|
|
||||||
stream.on('end', next);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
customTarOperations();
|
await tarTools.addFileToPack(pack, {
|
||||||
|
fileName: 'data.json',
|
||||||
|
content: Buffer.from(JSON.stringify({ foo: 'bar' }))
|
||||||
|
});
|
||||||
|
|
||||||
|
pack.finalize();
|
||||||
|
pack.pipe(createWriteStream('./output.tar'));
|
||||||
|
|
||||||
|
// Pack a directory to TAR.GZ buffer
|
||||||
|
const tgzBuffer = await tarTools.packDirectoryToTarGz('./src', 6);
|
||||||
|
|
||||||
|
// Pack a directory to TAR.GZ stream
|
||||||
|
const tgzStream = await tarTools.packDirectoryToTarGzStream('./src');
|
||||||
```
|
```
|
||||||
|
|
||||||
### Extract and Analyze All-in-One
|
### Working with ZIP archives directly
|
||||||
|
|
||||||
To extract and simultaneously analyze archive content:
|
```typescript
|
||||||
|
import { ZipTools } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
const zipTools = new ZipTools();
|
||||||
|
|
||||||
|
// Create a ZIP archive from entries
|
||||||
|
const zipBuffer = await zipTools.createZip([
|
||||||
|
{ archivePath: 'readme.txt', content: 'Hello!' },
|
||||||
|
{ archivePath: 'data.bin', content: Buffer.from([0x00, 0x01, 0x02]) }
|
||||||
|
], 6);
|
||||||
|
|
||||||
|
// Extract a ZIP buffer
|
||||||
|
const entries = await zipTools.extractZip(zipBuffer);
|
||||||
|
for (const entry of entries) {
|
||||||
|
console.log(`${entry.path}: ${entry.content.length} bytes`);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### In-memory round-trip
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { createReadStream, createWriteStream } from 'fs';
|
|
||||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
async function extractAndAnalyze() {
|
// Create archive in memory
|
||||||
const filePath = '/path/to/archive.zip';
|
const archive = await SmartArchive.create()
|
||||||
const targetDir = '/path/to/extract';
|
.format('tar.gz')
|
||||||
|
.entry('config.json', JSON.stringify({ version: '1.0.0' }))
|
||||||
|
.build();
|
||||||
|
|
||||||
const archive = await SmartArchive.fromArchiveFile(filePath);
|
const buffer = await archive.toBuffer();
|
||||||
const analyzedStream = archive.archiveAnalyzer.getAnalyzedStream();
|
|
||||||
const extractionStream = await archive.exportToStreamOfStreamFiles();
|
|
||||||
|
|
||||||
analyzedStream.pipe(extractionStream).pipe(createWriteStream(targetDir));
|
// Extract from buffer
|
||||||
|
const files = await SmartArchive.create()
|
||||||
|
.buffer(buffer)
|
||||||
|
.toSmartFiles();
|
||||||
|
|
||||||
analyzedStream.on('data', (chunk) => {
|
for (const file of files) {
|
||||||
console.log(JSON.stringify(chunk, null, 2));
|
console.log(`${file.relative}: ${file.contents.toString()}`);
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
extractAndAnalyze();
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Final Words
|
## Real-World Use Cases 🌍
|
||||||
|
|
||||||
These examples demonstrate various use cases for `@push.rocks/smartarchive`. Depending on your specific project requirements, you can adapt these examples to suit your needs. Always refer to the latest documentation for the most current information and methods available in `@push.rocks/smartarchive`.
|
### CI/CD: Download & Extract Build Artifacts
|
||||||
|
|
||||||
For more information and API references, check the official [`@push.rocks/smartarchive` GitHub repository](https://code.foss.global/push.rocks/smartarchive).
|
```typescript
|
||||||
|
const artifacts = await SmartArchive.create()
|
||||||
|
.url(`${CI_SERVER}/artifacts/build-${BUILD_ID}.zip`)
|
||||||
|
.stripComponents(1)
|
||||||
|
.extract('./dist');
|
||||||
|
```
|
||||||
|
|
||||||
|
### Backup System
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Create backup
|
||||||
|
await SmartArchive.create()
|
||||||
|
.format('tar.gz')
|
||||||
|
.compression(9)
|
||||||
|
.directory('./data')
|
||||||
|
.toFile(`./backups/backup-${Date.now()}.tar.gz`);
|
||||||
|
|
||||||
|
// Restore backup
|
||||||
|
await SmartArchive.create()
|
||||||
|
.file('./backups/backup-latest.tar.gz')
|
||||||
|
.extract('/restore/location');
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bundle files for HTTP download
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
// Express/Fastify handler
|
||||||
|
app.get('/download-bundle', async (req, res) => {
|
||||||
|
const buffer = await SmartArchive.create()
|
||||||
|
.format('zip')
|
||||||
|
.entry('report.pdf', pdfBuffer)
|
||||||
|
.entry('data.xlsx', excelBuffer)
|
||||||
|
.entry('images/chart.png', chartBuffer)
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
res.setHeader('Content-Type', 'application/zip');
|
||||||
|
res.setHeader('Content-Disposition', 'attachment; filename=report-bundle.zip');
|
||||||
|
res.send(buffer);
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data Pipeline: Process Compressed Datasets
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const fileStream = await SmartArchive.create()
|
||||||
|
.url('https://data.source/dataset.tar.gz')
|
||||||
|
.toStreamFiles();
|
||||||
|
|
||||||
|
fileStream.on('data', async (file) => {
|
||||||
|
if (file.relativeFilePath.endsWith('.csv')) {
|
||||||
|
const content = await file.getContentAsBuffer();
|
||||||
|
// Stream CSV processing...
|
||||||
|
}
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
## Supported Formats 📋
|
||||||
|
|
||||||
|
| Format | Extension(s) | Extract | Create |
|
||||||
|
|--------|--------------|---------|--------|
|
||||||
|
| TAR | `.tar` | ✅ | ✅ |
|
||||||
|
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ✅ |
|
||||||
|
| ZIP | `.zip` | ✅ | ✅ |
|
||||||
|
| GZIP | `.gz` | ✅ | ✅ |
|
||||||
|
| BZIP2 | `.bz2` | ✅ | ❌ |
|
||||||
|
|
||||||
|
## Type Definitions
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Supported archive formats
|
||||||
|
type TArchiveFormat = 'tar' | 'tar.gz' | 'tgz' | 'zip' | 'gz' | 'bz2';
|
||||||
|
|
||||||
|
// Compression level (0 = none, 9 = maximum)
|
||||||
|
type TCompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
|
||||||
|
|
||||||
|
// Entry for creating archives
|
||||||
|
interface IArchiveEntry {
|
||||||
|
archivePath: string;
|
||||||
|
content: string | Buffer | Readable | SmartFile | StreamFile;
|
||||||
|
size?: number;
|
||||||
|
mode?: number;
|
||||||
|
mtime?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Information about an archive entry
|
||||||
|
interface IArchiveEntryInfo {
|
||||||
|
path: string;
|
||||||
|
size: number;
|
||||||
|
isDirectory: boolean;
|
||||||
|
isFile: boolean;
|
||||||
|
mtime?: Date;
|
||||||
|
mode?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Archive analysis result
|
||||||
|
interface IArchiveInfo {
|
||||||
|
format: TArchiveFormat | null;
|
||||||
|
isCompressed: boolean;
|
||||||
|
isArchive: boolean;
|
||||||
|
entries?: IArchiveEntryInfo[];
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance Tips 🏎️
|
||||||
|
|
||||||
|
1. **Use streaming for large files** – `.toStreamFiles()` processes entries one at a time without loading the entire archive
|
||||||
|
2. **Provide byte lengths when known** – When using TarTools directly, provide `byteLength` for better performance
|
||||||
|
3. **Choose appropriate compression** – Use 1-3 for speed, 6 (default) for balance, 9 for maximum compression
|
||||||
|
4. **Filter early** – Use `.include()`/`.exclude()` to skip unwanted entries before processing
|
||||||
|
|
||||||
|
## Error Handling 🛡️
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||||
|
|
||||||
|
try {
|
||||||
|
await SmartArchive.create()
|
||||||
|
.url('https://example.com/file.zip')
|
||||||
|
.extract('./output');
|
||||||
|
} catch (error) {
|
||||||
|
if (error.message.includes('No source configured')) {
|
||||||
|
console.error('Forgot to specify source');
|
||||||
|
} else if (error.message.includes('No format specified')) {
|
||||||
|
console.error('Forgot to set format for creation');
|
||||||
|
} else if (error.message.includes('extraction mode')) {
|
||||||
|
console.error('Cannot mix extraction and creation methods');
|
||||||
|
} else {
|
||||||
|
console.error('Archive operation failed:', error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## License and Legal Information
|
## License and Legal Information
|
||||||
|
|
||||||
@@ -272,6 +532,10 @@ This repository contains open-source code that is licensed under the MIT License
|
|||||||
|
|
||||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||||
|
|
||||||
|
### Issue Reporting and Security
|
||||||
|
|
||||||
|
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||||
|
|
||||||
### Company Information
|
### Company Information
|
||||||
|
|
||||||
Task Venture Capital GmbH
|
Task Venture Capital GmbH
|
||||||
|
|||||||
@@ -1,13 +1,33 @@
|
|||||||
import * as path from 'path';
|
import * as path from 'node:path';
|
||||||
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsPromises from 'node:fs/promises';
|
||||||
import * as smartpath from '@push.rocks/smartpath';
|
import * as smartpath from '@push.rocks/smartpath';
|
||||||
import * as smartfile from '@push.rocks/smartfile';
|
import * as smartfile from '@push.rocks/smartfile';
|
||||||
import * as smartrequest from '@push.rocks/smartrequest';
|
import * as smartrequest from '@push.rocks/smartrequest';
|
||||||
import * as smartstream from '@push.rocks/smartstream';
|
import * as smartstream from '@push.rocks/smartstream';
|
||||||
|
|
||||||
export {
|
export { path, fs, fsPromises, smartpath, smartfile, smartrequest, smartstream };
|
||||||
path,
|
|
||||||
smartpath,
|
/**
|
||||||
smartfile,
|
* List files in a directory recursively, returning relative paths
|
||||||
smartrequest,
|
*/
|
||||||
smartstream,
|
export async function listFileTree(dirPath: string, _pattern: string = '**/*'): Promise<string[]> {
|
||||||
|
const results: string[] = [];
|
||||||
|
|
||||||
|
async function walkDir(currentPath: string, relativePath: string = '') {
|
||||||
|
const entries = await fsPromises.readdir(currentPath, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
const entryRelPath = relativePath ? path.join(relativePath, entry.name) : entry.name;
|
||||||
|
const entryFullPath = path.join(currentPath, entry.name);
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
await walkDir(entryFullPath, entryRelPath);
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
results.push(entryRelPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await walkDir(dirPath);
|
||||||
|
return results;
|
||||||
}
|
}
|
||||||
|
|||||||
382
test/test.gzip.node+deno.ts
Normal file
382
test/test.gzip.node+deno.ts
Normal file
@@ -0,0 +1,382 @@
|
|||||||
|
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||||
|
import * as plugins from './plugins.js';
|
||||||
|
import * as smartarchive from '../ts/index.js';
|
||||||
|
|
||||||
|
const testPaths = {
|
||||||
|
nogitDir: plugins.path.join(
|
||||||
|
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||||
|
'../.nogit/',
|
||||||
|
),
|
||||||
|
gzipTestDir: plugins.path.join(
|
||||||
|
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||||
|
'../.nogit/gzip-test',
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
tap.preTask('should prepare test directories', async () => {
|
||||||
|
await plugins.fsPromises.mkdir(testPaths.gzipTestDir, { recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should create and extract a gzip file', async () => {
|
||||||
|
// Create test data
|
||||||
|
const testContent = 'This is a test file for gzip compression and decompression.\n'.repeat(100);
|
||||||
|
const testFileName = 'test-file.txt';
|
||||||
|
const gzipFileName = 'test-file.txt.gz';
|
||||||
|
|
||||||
|
// Write the original file
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, testFileName),
|
||||||
|
testContent
|
||||||
|
);
|
||||||
|
|
||||||
|
// Create gzip compressed version using fflate directly
|
||||||
|
const fflate = await import('fflate');
|
||||||
|
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||||
|
Buffer.from(compressed)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Now test extraction using SmartArchive fluent API
|
||||||
|
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'extracted');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||||
|
.fileName('test-file.txt')
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
// Read the extracted file
|
||||||
|
const extractedContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, 'test-file.txt'),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify the content matches
|
||||||
|
expect(extractedContent).toEqual(testContent);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle gzip stream extraction', async () => {
|
||||||
|
// Create test data
|
||||||
|
const testContent = 'Stream test data for gzip\n'.repeat(50);
|
||||||
|
const gzipFileName = 'stream-test.txt.gz';
|
||||||
|
|
||||||
|
// Create gzip compressed version
|
||||||
|
const fflate = await import('fflate');
|
||||||
|
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||||
|
Buffer.from(compressed)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Create a read stream for the gzip file
|
||||||
|
const gzipStream = plugins.fs.createReadStream(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Test extraction using SmartArchive from stream with fluent API
|
||||||
|
const streamFiles: any[] = [];
|
||||||
|
const resultStream = await smartarchive.SmartArchive.create()
|
||||||
|
.stream(gzipStream)
|
||||||
|
.toStreamFiles();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
resultStream.on('data', (streamFile) => {
|
||||||
|
streamFiles.push(streamFile);
|
||||||
|
});
|
||||||
|
resultStream.on('end', resolve);
|
||||||
|
resultStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Verify we got the expected file
|
||||||
|
expect(streamFiles.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Read content from the stream file
|
||||||
|
if (streamFiles[0]) {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
const readStream = await streamFiles[0].createReadStream();
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
readStream.on('end', resolve);
|
||||||
|
readStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const extractedContent = Buffer.concat(chunks).toString();
|
||||||
|
expect(extractedContent).toEqual(testContent);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle gzip files with original filename in header', async () => {
|
||||||
|
// Test with a real-world gzip file that includes filename in header
|
||||||
|
const testContent = 'File with name in gzip header\n'.repeat(30);
|
||||||
|
const gzipFileName = 'compressed.gz';
|
||||||
|
|
||||||
|
// Create a proper gzip with filename header using Node's zlib
|
||||||
|
const zlib = await import('node:zlib');
|
||||||
|
const gzipBuffer = await new Promise<Buffer>((resolve, reject) => {
|
||||||
|
zlib.gzip(Buffer.from(testContent), {
|
||||||
|
level: 9,
|
||||||
|
}, (err, result) => {
|
||||||
|
if (err) reject(err);
|
||||||
|
else resolve(result);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||||
|
gzipBuffer
|
||||||
|
);
|
||||||
|
|
||||||
|
// Test extraction with fluent API
|
||||||
|
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'header-test');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||||
|
.fileName('compressed.txt')
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
// Check if file was extracted (name might be derived from archive name)
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Read and verify content
|
||||||
|
const extractedFile = files[0];
|
||||||
|
const extractedContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, extractedFile || 'compressed.txt'),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
expect(extractedContent).toEqual(testContent);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle large gzip files', async () => {
|
||||||
|
// Create a larger test file
|
||||||
|
const largeContent = 'x'.repeat(1024 * 1024); // 1MB of 'x' characters
|
||||||
|
const gzipFileName = 'large-file.txt.gz';
|
||||||
|
|
||||||
|
// Compress the large file
|
||||||
|
const fflate = await import('fflate');
|
||||||
|
const compressed = fflate.gzipSync(Buffer.from(largeContent));
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||||
|
Buffer.from(compressed)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Test extraction with fluent API
|
||||||
|
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'large-extracted');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||||
|
.fileName('large-file.txt')
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
// Verify the extracted content
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
const extractedContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, files[0] || 'large-file.txt'),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
expect(extractedContent.length).toEqual(largeContent.length);
|
||||||
|
expect(extractedContent).toEqual(largeContent);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle real-world multi-chunk gzip from URL', async () => {
|
||||||
|
// Test with a real tgz file that will be processed in multiple chunks
|
||||||
|
const testUrl = 'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz';
|
||||||
|
|
||||||
|
// Download and extract the archive with fluent API
|
||||||
|
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'real-world-test');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.url(testUrl)
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
// Verify extraction worked
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Check for expected package structure
|
||||||
|
const hasPackageJson = files.some(f => f.includes('package.json'));
|
||||||
|
expect(hasPackageJson).toBeTrue();
|
||||||
|
|
||||||
|
// Read and verify package.json content
|
||||||
|
const packageJsonPath = files.find(f => f.includes('package.json'));
|
||||||
|
if (packageJsonPath) {
|
||||||
|
const packageJsonContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, packageJsonPath),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
const packageJson = JSON.parse(packageJsonContent);
|
||||||
|
expect(packageJson.name).toEqual('@push.rocks/smartfile');
|
||||||
|
expect(packageJson.version).toEqual('11.2.7');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read and verify a TypeScript file
|
||||||
|
const tsFilePath = files.find(f => f.endsWith('.ts'));
|
||||||
|
if (tsFilePath) {
|
||||||
|
const tsFileContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, tsFilePath),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
// TypeScript files should have content
|
||||||
|
expect(tsFileContent.length).toBeGreaterThan(10);
|
||||||
|
console.log(` ✓ TypeScript file ${tsFilePath} has ${tsFileContent.length} bytes`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read and verify license file
|
||||||
|
const licensePath = files.find(f => f.includes('license'));
|
||||||
|
if (licensePath) {
|
||||||
|
const licenseContent = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, licensePath),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
expect(licenseContent).toContain('MIT');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify we can read multiple files without corruption
|
||||||
|
const readableFiles = files.filter(f =>
|
||||||
|
f.endsWith('.json') || f.endsWith('.md') || f.endsWith('.ts') || f.endsWith('.js')
|
||||||
|
).slice(0, 5); // Test first 5 readable files
|
||||||
|
|
||||||
|
for (const file of readableFiles) {
|
||||||
|
const content = await plugins.fsPromises.readFile(
|
||||||
|
plugins.path.join(extractPath, file),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
expect(content).toBeDefined();
|
||||||
|
expect(content.length).toBeGreaterThan(0);
|
||||||
|
console.log(` ✓ Successfully read ${file} (${content.length} bytes)`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle gzip extraction fully in memory', async () => {
|
||||||
|
// Create test data in memory
|
||||||
|
const testContent = 'This is test data for in-memory gzip processing\n'.repeat(100);
|
||||||
|
|
||||||
|
// Compress using fflate in memory
|
||||||
|
const fflate = await import('fflate');
|
||||||
|
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||||
|
|
||||||
|
// Process through SmartArchive without touching filesystem using fluent API
|
||||||
|
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||||
|
const resultStream = await smartarchive.SmartArchive.create()
|
||||||
|
.buffer(Buffer.from(compressed))
|
||||||
|
.toStreamFiles();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||||
|
streamFiles.push(streamFile);
|
||||||
|
});
|
||||||
|
resultStream.on('end', resolve);
|
||||||
|
resultStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Verify we got a file
|
||||||
|
expect(streamFiles.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Read the content from memory without filesystem
|
||||||
|
const firstFile = streamFiles[0];
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
const readStream = await firstFile.createReadStream();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
readStream.on('end', resolve);
|
||||||
|
readStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const extractedContent = Buffer.concat(chunks).toString();
|
||||||
|
expect(extractedContent).toEqual(testContent);
|
||||||
|
console.log(` ✓ In-memory extraction successful (${extractedContent.length} bytes)`);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle real tgz file fully in memory', async (tools) => {
|
||||||
|
await tools.timeout(10000); // Set 10 second timeout
|
||||||
|
// Download tgz file into memory
|
||||||
|
const response = await plugins.smartrequest.SmartRequest.create()
|
||||||
|
.url('https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz')
|
||||||
|
.get();
|
||||||
|
|
||||||
|
const tgzBuffer = Buffer.from(await response.arrayBuffer());
|
||||||
|
console.log(` Downloaded ${tgzBuffer.length} bytes into memory`);
|
||||||
|
|
||||||
|
// Process through SmartArchive in memory with fluent API
|
||||||
|
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||||
|
const resultStream = await smartarchive.SmartArchive.create()
|
||||||
|
.buffer(tgzBuffer)
|
||||||
|
.toStreamFiles();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
let timeout: NodeJS.Timeout;
|
||||||
|
|
||||||
|
const cleanup = () => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
};
|
||||||
|
|
||||||
|
timeout = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
resolve(); // Resolve after timeout if stream doesn't end
|
||||||
|
}, 5000);
|
||||||
|
|
||||||
|
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||||
|
streamFiles.push(streamFile);
|
||||||
|
});
|
||||||
|
|
||||||
|
resultStream.on('end', () => {
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
|
||||||
|
resultStream.on('error', (err) => {
|
||||||
|
cleanup();
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(` Extracted ${streamFiles.length} files in memory`);
|
||||||
|
// At minimum we should have extracted something
|
||||||
|
expect(streamFiles.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Find and read package.json from memory
|
||||||
|
const packageJsonFile = streamFiles.find(f => f.relativeFilePath?.includes('package.json'));
|
||||||
|
|
||||||
|
if (packageJsonFile) {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
const readStream = await packageJsonFile.createReadStream();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
readStream.on('end', resolve);
|
||||||
|
readStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const packageJsonContent = Buffer.concat(chunks).toString();
|
||||||
|
const packageJson = JSON.parse(packageJsonContent);
|
||||||
|
expect(packageJson.name).toEqual('@push.rocks/smartfile');
|
||||||
|
expect(packageJson.version).toEqual('11.2.7');
|
||||||
|
console.log(` ✓ Read package.json from memory: ${packageJson.name}@${packageJson.version}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read a few more files to verify integrity
|
||||||
|
const filesToCheck = streamFiles.slice(0, 3);
|
||||||
|
for (const file of filesToCheck) {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
const readStream = await file.createReadStream();
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
readStream.on('end', resolve);
|
||||||
|
readStream.on('error', reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = Buffer.concat(chunks);
|
||||||
|
expect(content.length).toBeGreaterThan(0);
|
||||||
|
console.log(` ✓ Read ${file.relativeFilePath} from memory (${content.length} bytes)`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
240
test/test.node+deno.ts
Normal file
240
test/test.node+deno.ts
Normal file
@@ -0,0 +1,240 @@
|
|||||||
|
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||||
|
|
||||||
|
import * as plugins from './plugins.js';
|
||||||
|
|
||||||
|
const testPaths = {
|
||||||
|
nogitDir: plugins.path.join(
|
||||||
|
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||||
|
'../.nogit/',
|
||||||
|
),
|
||||||
|
remoteDir: plugins.path.join(
|
||||||
|
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||||
|
'../.nogit/remote',
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
import * as smartarchive from '../ts/index.js';
|
||||||
|
|
||||||
|
tap.preTask('should prepare .nogit dir', async () => {
|
||||||
|
await plugins.fsPromises.mkdir(testPaths.remoteDir, { recursive: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.preTask('should prepare downloads', async (tools) => {
|
||||||
|
const response = await plugins.smartrequest.SmartRequest.create()
|
||||||
|
.url(
|
||||||
|
'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz',
|
||||||
|
)
|
||||||
|
.get();
|
||||||
|
const downloadedFile: Buffer = Buffer.from(await response.arrayBuffer());
|
||||||
|
await plugins.fsPromises.writeFile(
|
||||||
|
plugins.path.join(testPaths.nogitDir, 'test.tgz'),
|
||||||
|
downloadedFile,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract existing files on disk using fluent API', async () => {
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.extract(testPaths.nogitDir);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract from file using fluent API', async () => {
|
||||||
|
const extractPath = plugins.path.join(testPaths.nogitDir, 'from-file-test');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract with stripComponents using fluent API', async () => {
|
||||||
|
const extractPath = plugins.path.join(testPaths.nogitDir, 'strip-test');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.stripComponents(1)
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files.length).toBeGreaterThan(0);
|
||||||
|
// Files should not have 'package/' prefix
|
||||||
|
const hasPackagePrefix = files.some(f => f.startsWith('package/'));
|
||||||
|
expect(hasPackagePrefix).toBeFalse();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract with filter using fluent API', async () => {
|
||||||
|
const extractPath = plugins.path.join(testPaths.nogitDir, 'filter-test');
|
||||||
|
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.filter(entry => entry.path.endsWith('.json'))
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
// All extracted files should be JSON
|
||||||
|
for (const file of files) {
|
||||||
|
expect(file.endsWith('.json')).toBeTrue();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should list archive entries using fluent API', async () => {
|
||||||
|
const entries = await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.list();
|
||||||
|
|
||||||
|
expect(entries.length).toBeGreaterThan(0);
|
||||||
|
const hasPackageJson = entries.some(e => e.path.includes('package.json'));
|
||||||
|
expect(hasPackageJson).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should create archive using fluent API', async () => {
|
||||||
|
const archive = await smartarchive.SmartArchive.create()
|
||||||
|
.format('tar.gz')
|
||||||
|
.compression(9)
|
||||||
|
.entry('hello.txt', 'Hello World!')
|
||||||
|
.entry('config.json', JSON.stringify({ name: 'test', version: '1.0.0' }));
|
||||||
|
|
||||||
|
expect(archive).toBeInstanceOf(smartarchive.SmartArchive);
|
||||||
|
|
||||||
|
const buffer = await archive.toBuffer();
|
||||||
|
expect(buffer.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should create and write archive to file using fluent API', async () => {
|
||||||
|
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.tar.gz');
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.format('tar.gz')
|
||||||
|
.entry('readme.txt', 'This is a test archive')
|
||||||
|
.entry('data/info.json', JSON.stringify({ created: new Date().toISOString() }))
|
||||||
|
.toFile(outputPath);
|
||||||
|
|
||||||
|
// Verify file was created
|
||||||
|
const stats = await plugins.fsPromises.stat(outputPath);
|
||||||
|
expect(stats.size).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// Verify we can extract it
|
||||||
|
const extractPath = plugins.path.join(testPaths.nogitDir, 'verify-created');
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.file(outputPath)
|
||||||
|
.extract(extractPath);
|
||||||
|
|
||||||
|
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||||
|
expect(files).toContain('readme.txt');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should create ZIP archive using fluent API', async () => {
|
||||||
|
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.zip');
|
||||||
|
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.format('zip')
|
||||||
|
.entry('file1.txt', 'Content 1')
|
||||||
|
.entry('file2.txt', 'Content 2')
|
||||||
|
.toFile(outputPath);
|
||||||
|
|
||||||
|
// Verify file was created
|
||||||
|
const stats = await plugins.fsPromises.stat(outputPath);
|
||||||
|
expect(stats.size).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract to SmartFiles using fluent API', async () => {
|
||||||
|
const smartFiles = await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.toSmartFiles();
|
||||||
|
|
||||||
|
expect(smartFiles.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
const packageJson = smartFiles.find(f => f.relative.includes('package.json'));
|
||||||
|
expect(packageJson).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should analyze archive using fluent API', async () => {
|
||||||
|
const info = await smartarchive.SmartArchive.create()
|
||||||
|
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
|
||||||
|
.analyze();
|
||||||
|
|
||||||
|
expect(info.isArchive).toBeTrue();
|
||||||
|
expect(info.isCompressed).toBeTrue();
|
||||||
|
expect(info.format).toEqual('gz');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should check if file exists in archive using fluent API', async () => {
|
||||||
|
const hasPackageJson = await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.hasFile('package.json');
|
||||||
|
|
||||||
|
expect(hasPackageJson).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should extract single file using fluent API', async () => {
|
||||||
|
const packageJson = await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.extractFile('package.json');
|
||||||
|
|
||||||
|
expect(packageJson).toBeDefined();
|
||||||
|
expect(packageJson!.contents.toString()).toContain('websetup');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should handle include/exclude patterns', async () => {
|
||||||
|
const smartFiles = await smartarchive.SmartArchive.create()
|
||||||
|
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||||
|
.include(/\.json$/)
|
||||||
|
.toSmartFiles();
|
||||||
|
|
||||||
|
expect(smartFiles.length).toBeGreaterThan(0);
|
||||||
|
for (const file of smartFiles) {
|
||||||
|
expect(file.relative.endsWith('.json')).toBeTrue();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error when mixing modes', async () => {
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
smartarchive.SmartArchive.create()
|
||||||
|
.url('https://example.com/archive.tgz')
|
||||||
|
.entry('file.txt', 'content'); // This should throw
|
||||||
|
} catch (e) {
|
||||||
|
threw = true;
|
||||||
|
expect((e as Error).message).toContain('extraction mode');
|
||||||
|
}
|
||||||
|
expect(threw).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error when no source configured', async () => {
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await smartarchive.SmartArchive.create().extract('./output');
|
||||||
|
} catch (e) {
|
||||||
|
threw = true;
|
||||||
|
expect((e as Error).message).toContain('No source configured');
|
||||||
|
}
|
||||||
|
expect(threw).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should throw error when no format configured', async () => {
|
||||||
|
let threw = false;
|
||||||
|
try {
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.entry('file.txt', 'content')
|
||||||
|
.toBuffer();
|
||||||
|
} catch (e) {
|
||||||
|
threw = true;
|
||||||
|
expect((e as Error).message).toContain('No format specified');
|
||||||
|
}
|
||||||
|
expect(threw).toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.skip.test('should extract a b2zip', async () => {
|
||||||
|
const dataUrl =
|
||||||
|
'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||||||
|
await smartarchive.SmartArchive.create()
|
||||||
|
.url(dataUrl)
|
||||||
|
.extract(plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'));
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
50
test/test.ts
50
test/test.ts
@@ -1,50 +0,0 @@
|
|||||||
import { tap, expect } from '@push.rocks/tapbundle';
|
|
||||||
|
|
||||||
import * as plugins from './plugins.js';
|
|
||||||
|
|
||||||
const testPaths = {
|
|
||||||
nogitDir: plugins.path.join(
|
|
||||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
|
||||||
'../.nogit/'
|
|
||||||
),
|
|
||||||
remoteDir: plugins.path.join(
|
|
||||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
|
||||||
'../.nogit/remote'
|
|
||||||
),
|
|
||||||
};
|
|
||||||
|
|
||||||
import * as smartarchive from '../ts/index.js';
|
|
||||||
|
|
||||||
tap.preTask('should prepare .nogit dir', async () => {
|
|
||||||
await plugins.smartfile.fs.ensureDir(testPaths.remoteDir);
|
|
||||||
});
|
|
||||||
|
|
||||||
tap.preTask('should prepare downloads', async (tools) => {
|
|
||||||
const downloadedFile: Buffer = (
|
|
||||||
await plugins.smartrequest.getBinary(
|
|
||||||
'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
|
|
||||||
)
|
|
||||||
).body;
|
|
||||||
await plugins.smartfile.memory.toFs(
|
|
||||||
downloadedFile,
|
|
||||||
plugins.path.join(testPaths.nogitDir, 'test.tgz')
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
tap.test('should extract existing files on disk', async () => {
|
|
||||||
const testSmartarchive = await smartarchive.SmartArchive.fromArchiveUrl(
|
|
||||||
'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz'
|
|
||||||
);
|
|
||||||
await testSmartarchive.exportToFs(testPaths.nogitDir);
|
|
||||||
});
|
|
||||||
|
|
||||||
tap.skip.test('should extract a b2zip', async () => {
|
|
||||||
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
|
||||||
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
|
||||||
await testArchive.exportToFs(
|
|
||||||
plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'),
|
|
||||||
'data.jsonl',
|
|
||||||
);
|
|
||||||
})
|
|
||||||
|
|
||||||
await tap.start();
|
|
||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartarchive',
|
name: '@push.rocks/smartarchive',
|
||||||
version: '4.0.38',
|
version: '5.0.1',
|
||||||
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
|
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,41 +1,60 @@
|
|||||||
var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF];
|
import type { IBitReader } from '../interfaces.js';
|
||||||
|
|
||||||
// returns a function that reads bits.
|
const BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff] as const;
|
||||||
// takes a buffer iterator as input
|
|
||||||
export function bitIterator(nextBuffer: () => Buffer) {
|
/**
|
||||||
var bit = 0, byte = 0;
|
* Creates a bit reader function for BZIP2 decompression.
|
||||||
var bytes = nextBuffer();
|
* Takes a buffer iterator as input and returns a function that reads bits.
|
||||||
var f = function(n) {
|
*/
|
||||||
if (n === null && bit != 0) { // align to byte boundary
|
export function bitIterator(nextBuffer: () => Buffer): IBitReader {
|
||||||
bit = 0
|
let bit = 0;
|
||||||
byte++;
|
let byte = 0;
|
||||||
return;
|
let bytes = nextBuffer();
|
||||||
}
|
let _bytesRead = 0;
|
||||||
var result = 0;
|
|
||||||
while(n > 0) {
|
const reader = function (n: number | null): number | void {
|
||||||
if (byte >= bytes.length) {
|
if (n === null && bit !== 0) {
|
||||||
byte = 0;
|
// align to byte boundary
|
||||||
bytes = nextBuffer();
|
bit = 0;
|
||||||
}
|
byte++;
|
||||||
var left = 8 - bit;
|
return;
|
||||||
if (bit === 0 && n > 0)
|
}
|
||||||
// @ts-ignore
|
|
||||||
f.bytesRead++;
|
let result = 0;
|
||||||
if (n >= left) {
|
let remaining = n as number;
|
||||||
result <<= left;
|
|
||||||
result |= (BITMASK[left] & bytes[byte++]);
|
while (remaining > 0) {
|
||||||
bit = 0;
|
if (byte >= bytes.length) {
|
||||||
n -= left;
|
byte = 0;
|
||||||
} else {
|
bytes = nextBuffer();
|
||||||
result <<= n;
|
}
|
||||||
result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit));
|
|
||||||
bit += n;
|
const left = 8 - bit;
|
||||||
n = 0;
|
|
||||||
}
|
if (bit === 0 && remaining > 0) {
|
||||||
}
|
_bytesRead++;
|
||||||
return result;
|
}
|
||||||
};
|
|
||||||
// @ts-ignore
|
if (remaining >= left) {
|
||||||
f.bytesRead = 0;
|
result <<= left;
|
||||||
return f;
|
result |= BITMASK[left] & bytes[byte++];
|
||||||
};
|
bit = 0;
|
||||||
|
remaining -= left;
|
||||||
|
} else {
|
||||||
|
result <<= remaining;
|
||||||
|
result |= (bytes[byte] & (BITMASK[remaining] << (8 - remaining - bit))) >> (8 - remaining - bit);
|
||||||
|
bit += remaining;
|
||||||
|
remaining = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
} as IBitReader;
|
||||||
|
|
||||||
|
Object.defineProperty(reader, 'bytesRead', {
|
||||||
|
get: () => _bytesRead,
|
||||||
|
enumerable: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
return reader;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,335 +1,449 @@
|
|||||||
export class Bzip2Error extends Error {
|
import { Bzip2Error, BZIP2_ERROR_CODES } from '../errors.js';
|
||||||
public name: string = 'Bzip2Error';
|
import type { IBitReader, IHuffmanGroup } from '../interfaces.js';
|
||||||
public message: string;
|
|
||||||
public stack = (new Error()).stack;
|
|
||||||
|
|
||||||
constructor(messageArg: string) {
|
// Re-export Bzip2Error for backward compatibility
|
||||||
super();
|
export { Bzip2Error };
|
||||||
this.message = messageArg;
|
|
||||||
}
|
/**
|
||||||
|
* Throw a BZIP2 error with proper error code
|
||||||
|
*/
|
||||||
|
function throwError(message: string, code: string = BZIP2_ERROR_CODES.INVALID_BLOCK_DATA): never {
|
||||||
|
throw new Bzip2Error(message, code);
|
||||||
}
|
}
|
||||||
|
|
||||||
var messageArg = {
|
/**
|
||||||
Error: function(message) {throw new Bzip2Error(message);}
|
* BZIP2 decompression implementation
|
||||||
};
|
*/
|
||||||
|
|
||||||
export class Bzip2 {
|
export class Bzip2 {
|
||||||
public Bzip2Error = Bzip2Error;
|
// CRC32 lookup table for BZIP2
|
||||||
public crcTable =
|
public readonly crcTable: readonly number[] = [
|
||||||
[
|
0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b,
|
||||||
0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9,
|
0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
|
||||||
0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005,
|
0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7,
|
||||||
0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
|
0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
|
||||||
0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd,
|
0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3,
|
||||||
0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9,
|
0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039,
|
||||||
0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
|
0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef,
|
||||||
0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011,
|
0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
|
||||||
0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd,
|
0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb,
|
||||||
0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039,
|
0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
|
||||||
0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5,
|
0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0,
|
||||||
0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81,
|
0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
|
||||||
0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
|
0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4,
|
||||||
0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49,
|
0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde,
|
||||||
0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95,
|
0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08,
|
||||||
0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
|
0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
|
||||||
0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d,
|
0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc,
|
||||||
0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae,
|
0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6,
|
||||||
0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
|
0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050,
|
||||||
0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16,
|
0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
|
||||||
0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca,
|
0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34,
|
||||||
0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde,
|
0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637,
|
||||||
0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02,
|
0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1,
|
||||||
0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066,
|
0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
|
||||||
0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
|
0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5,
|
||||||
0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e,
|
0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff,
|
||||||
0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692,
|
0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9,
|
||||||
0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6,
|
0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
|
||||||
0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a,
|
0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd,
|
||||||
0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e,
|
0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
|
||||||
0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
|
0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71,
|
||||||
0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686,
|
0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
|
||||||
0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a,
|
0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2,
|
||||||
0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637,
|
0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8,
|
||||||
0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb,
|
0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e,
|
||||||
0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f,
|
0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
|
||||||
0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
|
0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a,
|
||||||
0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47,
|
0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0,
|
||||||
0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b,
|
0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676,
|
||||||
0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff,
|
0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
|
||||||
0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623,
|
0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662,
|
||||||
0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7,
|
0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
|
||||||
0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
|
0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4,
|
||||||
0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f,
|
|
||||||
0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3,
|
|
||||||
0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
|
|
||||||
0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b,
|
|
||||||
0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f,
|
|
||||||
0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
|
|
||||||
0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640,
|
|
||||||
0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c,
|
|
||||||
0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8,
|
|
||||||
0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24,
|
|
||||||
0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30,
|
|
||||||
0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
|
|
||||||
0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088,
|
|
||||||
0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654,
|
|
||||||
0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0,
|
|
||||||
0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c,
|
|
||||||
0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18,
|
|
||||||
0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
|
|
||||||
0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0,
|
|
||||||
0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c,
|
|
||||||
0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
|
|
||||||
0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4
|
|
||||||
];
|
];
|
||||||
|
|
||||||
array = function(bytes) {
|
// State arrays initialized in header()
|
||||||
var bit = 0, byte = 0;
|
private byteCount!: Int32Array;
|
||||||
var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ];
|
private symToByte!: Uint8Array;
|
||||||
return function(n) {
|
private mtfSymbol!: Int32Array;
|
||||||
var result = 0;
|
private selectors!: Uint8Array;
|
||||||
while(n > 0) {
|
|
||||||
var left = 8 - bit;
|
/**
|
||||||
if (n >= left) {
|
* Create a bit reader from a byte array
|
||||||
result <<= left;
|
*/
|
||||||
result |= (BITMASK[left] & bytes[byte++]);
|
array(bytes: Uint8Array | Buffer): (n: number) => number {
|
||||||
bit = 0;
|
let bit = 0;
|
||||||
n -= left;
|
let byte = 0;
|
||||||
} else {
|
const BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff];
|
||||||
result <<= n;
|
|
||||||
result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit));
|
return function (n: number): number {
|
||||||
bit += n;
|
let result = 0;
|
||||||
n = 0;
|
while (n > 0) {
|
||||||
}
|
const left = 8 - bit;
|
||||||
|
if (n >= left) {
|
||||||
|
result <<= left;
|
||||||
|
result |= BITMASK[left] & bytes[byte++];
|
||||||
|
bit = 0;
|
||||||
|
n -= left;
|
||||||
|
} else {
|
||||||
|
result <<= n;
|
||||||
|
result |= (bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit);
|
||||||
|
bit += n;
|
||||||
|
n = 0;
|
||||||
}
|
}
|
||||||
return result;
|
}
|
||||||
}
|
return result;
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
simple = function(srcbuffer, stream) {
|
/**
|
||||||
var bits = this.array(srcbuffer);
|
* Simple decompression from a buffer
|
||||||
var size = this.header(bits);
|
*/
|
||||||
var ret = false;
|
simple(srcbuffer: Uint8Array | Buffer, stream: (byte: number) => void): void {
|
||||||
var bufsize = 100000 * size;
|
const bits = this.array(srcbuffer);
|
||||||
var buf = new Int32Array(bufsize);
|
const size = this.header(bits as IBitReader);
|
||||||
|
let ret: number | null = 0;
|
||||||
|
const bufsize = 100000 * size;
|
||||||
|
const buf = new Int32Array(bufsize);
|
||||||
|
|
||||||
do {
|
do {
|
||||||
ret = this.decompress(bits, stream, buf, bufsize);
|
ret = this.decompress(bits as IBitReader, stream, buf, bufsize, ret);
|
||||||
} while(!ret);
|
} while (ret !== null);
|
||||||
}
|
}
|
||||||
|
|
||||||
header = function(bits) {
|
/**
|
||||||
|
* Parse BZIP2 header and return block size
|
||||||
|
*/
|
||||||
|
header(bits: IBitReader): number {
|
||||||
this.byteCount = new Int32Array(256);
|
this.byteCount = new Int32Array(256);
|
||||||
this.symToByte = new Uint8Array(256);
|
this.symToByte = new Uint8Array(256);
|
||||||
this.mtfSymbol = new Int32Array(256);
|
this.mtfSymbol = new Int32Array(256);
|
||||||
this.selectors = new Uint8Array(0x8000);
|
this.selectors = new Uint8Array(0x8000);
|
||||||
|
|
||||||
if (bits(8*3) != 4348520) messageArg.Error("No magic number found");
|
if (bits(8 * 3) !== 4348520) {
|
||||||
|
throwError('No BZIP2 magic number found at start of stream', BZIP2_ERROR_CODES.NO_MAGIC_NUMBER);
|
||||||
|
}
|
||||||
|
|
||||||
var i = bits(8) - 48;
|
const blockSize = (bits(8) as number) - 48;
|
||||||
if (i < 1 || i > 9) messageArg.Error("Not a BZIP archive");
|
if (blockSize < 1 || blockSize > 9) {
|
||||||
return i;
|
throwError('Invalid BZIP2 archive: block size must be 1-9', BZIP2_ERROR_CODES.INVALID_ARCHIVE);
|
||||||
};
|
}
|
||||||
|
return blockSize;
|
||||||
|
}
|
||||||
|
|
||||||
decompress = function(bits, stream, buf, bufsize, streamCRC) {
|
/**
|
||||||
var MAX_HUFCODE_BITS = 20;
|
* Decompress a BZIP2 block
|
||||||
var MAX_SYMBOLS = 258;
|
*/
|
||||||
var SYMBOL_RUNA = 0;
|
decompress(
|
||||||
var SYMBOL_RUNB = 1;
|
bits: IBitReader,
|
||||||
var GROUP_SIZE = 50;
|
stream: (byte: number) => void,
|
||||||
var crc = 0 ^ (-1);
|
buf: Int32Array,
|
||||||
|
bufsize: number,
|
||||||
|
streamCRC?: number | null
|
||||||
|
): number | null {
|
||||||
|
const MAX_HUFCODE_BITS = 20;
|
||||||
|
const MAX_SYMBOLS = 258;
|
||||||
|
const SYMBOL_RUNA = 0;
|
||||||
|
const SYMBOL_RUNB = 1;
|
||||||
|
const GROUP_SIZE = 50;
|
||||||
|
let crc = 0 ^ -1;
|
||||||
|
|
||||||
for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16);
|
// Read block header
|
||||||
if (h == "177245385090") {
|
let headerHex = '';
|
||||||
var finalCRC = bits(32)|0;
|
for (let i = 0; i < 6; i++) {
|
||||||
if (finalCRC !== streamCRC) messageArg.Error("Error in bzip2: crc32 do not match");
|
headerHex += (bits(8) as number).toString(16);
|
||||||
// align stream to byte
|
}
|
||||||
|
|
||||||
|
// Check for end-of-stream marker
|
||||||
|
if (headerHex === '177245385090') {
|
||||||
|
const finalCRC = bits(32) as number | 0;
|
||||||
|
if (finalCRC !== streamCRC) {
|
||||||
|
throwError('CRC32 mismatch: stream checksum verification failed', BZIP2_ERROR_CODES.CRC_MISMATCH);
|
||||||
|
}
|
||||||
|
// Align stream to byte boundary
|
||||||
bits(null);
|
bits(null);
|
||||||
return null; // reset streamCRC for next call
|
return null;
|
||||||
}
|
|
||||||
if (h != "314159265359") messageArg.Error("eek not valid bzip data");
|
|
||||||
var crcblock = bits(32)|0; // CRC code
|
|
||||||
if (bits(1)) messageArg.Error("unsupported obsolete version");
|
|
||||||
var origPtr = bits(24);
|
|
||||||
if (origPtr > bufsize) messageArg.Error("Initial position larger than buffer size");
|
|
||||||
var t = bits(16);
|
|
||||||
var symTotal = 0;
|
|
||||||
for (i = 0; i < 16; i++) {
|
|
||||||
if (t & (1 << (15 - i))) {
|
|
||||||
var k = bits(16);
|
|
||||||
for(j = 0; j < 16; j++) {
|
|
||||||
if (k & (1 << (15 - j))) {
|
|
||||||
this.symToByte[symTotal++] = (16 * i) + j;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var groupCount = bits(3);
|
// Verify block signature (pi digits)
|
||||||
if (groupCount < 2 || groupCount > 6) messageArg.Error("another error");
|
if (headerHex !== '314159265359') {
|
||||||
var nSelectors = bits(15);
|
throwError('Invalid block header: expected pi signature (0x314159265359)', BZIP2_ERROR_CODES.INVALID_BLOCK_DATA);
|
||||||
if (nSelectors == 0) messageArg.Error("meh");
|
|
||||||
for(var i = 0; i < groupCount; i++) this.mtfSymbol[i] = i;
|
|
||||||
|
|
||||||
for(var i = 0; i < nSelectors; i++) {
|
|
||||||
for(var j = 0; bits(1); j++) if (j >= groupCount) messageArg.Error("whoops another error");
|
|
||||||
var uc = this.mtfSymbol[j];
|
|
||||||
for(var k: any = j-1; k>=0; k--) {
|
|
||||||
this.mtfSymbol[k+1] = this.mtfSymbol[k];
|
|
||||||
}
|
|
||||||
this.mtfSymbol[0] = uc;
|
|
||||||
this.selectors[i] = uc;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var symCount = symTotal + 2;
|
const crcblock = bits(32) as number | 0;
|
||||||
var groups = [];
|
|
||||||
var length = new Uint8Array(MAX_SYMBOLS),
|
|
||||||
temp = new Uint16Array(MAX_HUFCODE_BITS+1);
|
|
||||||
|
|
||||||
var hufGroup;
|
if (bits(1)) {
|
||||||
|
throwError('Unsupported obsolete BZIP2 format version', BZIP2_ERROR_CODES.INVALID_ARCHIVE);
|
||||||
for(var j = 0; j < groupCount; j++) {
|
|
||||||
t = bits(5); //lengths
|
|
||||||
for(var i = 0; i < symCount; i++) {
|
|
||||||
while(true){
|
|
||||||
if (t < 1 || t > MAX_HUFCODE_BITS) messageArg.Error("I gave up a while ago on writing error messages");
|
|
||||||
if (!bits(1)) break;
|
|
||||||
if (!bits(1)) t++;
|
|
||||||
else t--;
|
|
||||||
}
|
|
||||||
length[i] = t;
|
|
||||||
}
|
|
||||||
var minLen, maxLen;
|
|
||||||
minLen = maxLen = length[0];
|
|
||||||
for(var i = 1; i < symCount; i++) {
|
|
||||||
if (length[i] > maxLen) maxLen = length[i];
|
|
||||||
else if (length[i] < minLen) minLen = length[i];
|
|
||||||
}
|
|
||||||
hufGroup = groups[j] = {};
|
|
||||||
hufGroup.permute = new Int32Array(MAX_SYMBOLS);
|
|
||||||
hufGroup.limit = new Int32Array(MAX_HUFCODE_BITS + 1);
|
|
||||||
hufGroup.base = new Int32Array(MAX_HUFCODE_BITS + 1);
|
|
||||||
|
|
||||||
hufGroup.minLen = minLen;
|
|
||||||
hufGroup.maxLen = maxLen;
|
|
||||||
var base = hufGroup.base;
|
|
||||||
var limit = hufGroup.limit;
|
|
||||||
var pp = 0;
|
|
||||||
for(var i: number = minLen; i <= maxLen; i++)
|
|
||||||
for(var t: any = 0; t < symCount; t++)
|
|
||||||
if (length[t] == i) hufGroup.permute[pp++] = t;
|
|
||||||
for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0;
|
|
||||||
for(i = 0; i < symCount; i++) temp[length[i]]++;
|
|
||||||
pp = t = 0;
|
|
||||||
for(i = minLen; i < maxLen; i++) {
|
|
||||||
pp += temp[i];
|
|
||||||
limit[i] = pp - 1;
|
|
||||||
pp <<= 1;
|
|
||||||
base[i+1] = pp - (t += temp[i]);
|
|
||||||
}
|
|
||||||
limit[maxLen] = pp + temp[maxLen] - 1;
|
|
||||||
base[minLen] = 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for(var i = 0; i < 256; i++) {
|
const origPtr = bits(24) as number;
|
||||||
this.mtfSymbol[i] = i;
|
if (origPtr > bufsize) {
|
||||||
this.byteCount[i] = 0;
|
throwError('Initial position larger than buffer size', BZIP2_ERROR_CODES.BUFFER_OVERFLOW);
|
||||||
}
|
}
|
||||||
var runPos, count, symCount: number, selector;
|
|
||||||
runPos = count = symCount = selector = 0;
|
// Read symbol map
|
||||||
while(true) {
|
let symbolMapBits = bits(16) as number;
|
||||||
if (!(symCount--)) {
|
let symTotal = 0;
|
||||||
symCount = GROUP_SIZE - 1;
|
for (let i = 0; i < 16; i++) {
|
||||||
if (selector >= nSelectors) messageArg.Error("meow i'm a kitty, that's an error");
|
if (symbolMapBits & (1 << (15 - i))) {
|
||||||
hufGroup = groups[this.selectors[selector++]];
|
const subMap = bits(16) as number;
|
||||||
base = hufGroup.base;
|
for (let j = 0; j < 16; j++) {
|
||||||
limit = hufGroup.limit;
|
if (subMap & (1 << (15 - j))) {
|
||||||
|
this.symToByte[symTotal++] = 16 * i + j;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
i = hufGroup.minLen;
|
}
|
||||||
j = bits(i);
|
|
||||||
while(true) {
|
|
||||||
if (i > hufGroup.maxLen) messageArg.Error("rawr i'm a dinosaur");
|
|
||||||
if (j <= limit[i]) break;
|
|
||||||
i++;
|
|
||||||
j = (j << 1) | bits(1);
|
|
||||||
}
|
|
||||||
j -= base[i];
|
|
||||||
if (j < 0 || j >= MAX_SYMBOLS) messageArg.Error("moo i'm a cow");
|
|
||||||
var nextSym = hufGroup.permute[j];
|
|
||||||
if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) {
|
|
||||||
if (!runPos){
|
|
||||||
runPos = 1;
|
|
||||||
t = 0;
|
|
||||||
}
|
|
||||||
if (nextSym == SYMBOL_RUNA) t += runPos;
|
|
||||||
else t += 2 * runPos;
|
|
||||||
runPos <<= 1;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (runPos) {
|
|
||||||
runPos = 0;
|
|
||||||
if (count + t > bufsize) messageArg.Error("Boom.");
|
|
||||||
uc = this.symToByte[this.mtfSymbol[0]];
|
|
||||||
this.byteCount[uc] += t;
|
|
||||||
while(t--) buf[count++] = uc;
|
|
||||||
}
|
|
||||||
if (nextSym > symTotal) break;
|
|
||||||
if (count >= bufsize) messageArg.Error("I can't think of anything. Error");
|
|
||||||
i = nextSym - 1;
|
|
||||||
uc = this.mtfSymbol[i];
|
|
||||||
for(var k: any = i-1; k>=0; k--) {
|
|
||||||
this.mtfSymbol[k+1] = this.mtfSymbol[k];
|
|
||||||
}
|
|
||||||
this.mtfSymbol[0] = uc
|
|
||||||
uc = this.symToByte[uc];
|
|
||||||
this.byteCount[uc]++;
|
|
||||||
buf[count++] = uc;
|
|
||||||
}
|
}
|
||||||
if (origPtr < 0 || origPtr >= count) messageArg.Error("I'm a monkey and I'm throwing something at someone, namely you");
|
|
||||||
var j = 0;
|
// Read Huffman groups
|
||||||
for(var i = 0; i < 256; i++) {
|
const groupCount = bits(3) as number;
|
||||||
k = j + this.byteCount[i];
|
if (groupCount < 2 || groupCount > 6) {
|
||||||
this.byteCount[i] = j;
|
throwError('Invalid group count: must be between 2 and 6', BZIP2_ERROR_CODES.INVALID_HUFFMAN);
|
||||||
j = k;
|
|
||||||
}
|
}
|
||||||
for(var i = 0; i < count; i++) {
|
|
||||||
uc = buf[i] & 0xff;
|
const nSelectors = bits(15) as number;
|
||||||
buf[this.byteCount[uc]] |= (i << 8);
|
if (nSelectors === 0) {
|
||||||
this.byteCount[uc]++;
|
throwError('Invalid selector count: cannot be zero', BZIP2_ERROR_CODES.INVALID_SELECTOR);
|
||||||
}
|
}
|
||||||
var pos = 0, current = 0, run = 0;
|
|
||||||
|
// Initialize MTF symbol array
|
||||||
|
for (let i = 0; i < groupCount; i++) {
|
||||||
|
this.mtfSymbol[i] = i;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read selectors using MTF decoding
|
||||||
|
for (let i = 0; i < nSelectors; i++) {
|
||||||
|
let j = 0;
|
||||||
|
while (bits(1)) {
|
||||||
|
j++;
|
||||||
|
if (j >= groupCount) {
|
||||||
|
throwError('Invalid MTF index: exceeds group count', BZIP2_ERROR_CODES.INVALID_HUFFMAN);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const uc = this.mtfSymbol[j];
|
||||||
|
for (let k = j - 1; k >= 0; k--) {
|
||||||
|
this.mtfSymbol[k + 1] = this.mtfSymbol[k];
|
||||||
|
}
|
||||||
|
this.mtfSymbol[0] = uc;
|
||||||
|
this.selectors[i] = uc;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build Huffman tables
|
||||||
|
const symCount = symTotal + 2;
|
||||||
|
const groups: IHuffmanGroup[] = [];
|
||||||
|
const length = new Uint8Array(MAX_SYMBOLS);
|
||||||
|
const temp = new Uint16Array(MAX_HUFCODE_BITS + 1);
|
||||||
|
|
||||||
|
for (let j = 0; j < groupCount; j++) {
|
||||||
|
let t = bits(5) as number;
|
||||||
|
for (let i = 0; i < symCount; i++) {
|
||||||
|
while (true) {
|
||||||
|
if (t < 1 || t > MAX_HUFCODE_BITS) {
|
||||||
|
throwError('Invalid Huffman code length: must be between 1 and 20', BZIP2_ERROR_CODES.INVALID_HUFFMAN);
|
||||||
|
}
|
||||||
|
if (!bits(1)) break;
|
||||||
|
if (!bits(1)) t++;
|
||||||
|
else t--;
|
||||||
|
}
|
||||||
|
length[i] = t;
|
||||||
|
}
|
||||||
|
|
||||||
|
let minLen = length[0];
|
||||||
|
let maxLen = length[0];
|
||||||
|
for (let i = 1; i < symCount; i++) {
|
||||||
|
if (length[i] > maxLen) maxLen = length[i];
|
||||||
|
else if (length[i] < minLen) minLen = length[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
const hufGroup: IHuffmanGroup = {
|
||||||
|
permute: new Int32Array(MAX_SYMBOLS),
|
||||||
|
limit: new Int32Array(MAX_HUFCODE_BITS + 1),
|
||||||
|
base: new Int32Array(MAX_HUFCODE_BITS + 1),
|
||||||
|
minLen,
|
||||||
|
maxLen,
|
||||||
|
};
|
||||||
|
groups[j] = hufGroup;
|
||||||
|
|
||||||
|
const base = hufGroup.base;
|
||||||
|
const limit = hufGroup.limit;
|
||||||
|
|
||||||
|
let pp = 0;
|
||||||
|
for (let i = minLen; i <= maxLen; i++) {
|
||||||
|
for (let t = 0; t < symCount; t++) {
|
||||||
|
if (length[t] === i) hufGroup.permute[pp++] = t;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = minLen; i <= maxLen; i++) {
|
||||||
|
temp[i] = 0;
|
||||||
|
limit[i] = 0;
|
||||||
|
}
|
||||||
|
for (let i = 0; i < symCount; i++) {
|
||||||
|
temp[length[i]]++;
|
||||||
|
}
|
||||||
|
|
||||||
|
pp = 0;
|
||||||
|
let tt = 0;
|
||||||
|
for (let i = minLen; i < maxLen; i++) {
|
||||||
|
pp += temp[i];
|
||||||
|
limit[i] = pp - 1;
|
||||||
|
pp <<= 1;
|
||||||
|
base[i + 1] = pp - (tt += temp[i]);
|
||||||
|
}
|
||||||
|
limit[maxLen] = pp + temp[maxLen] - 1;
|
||||||
|
base[minLen] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize for decoding
|
||||||
|
for (let i = 0; i < 256; i++) {
|
||||||
|
this.mtfSymbol[i] = i;
|
||||||
|
this.byteCount[i] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
let runPos = 0;
|
||||||
|
let count = 0;
|
||||||
|
let symCountRemaining = 0;
|
||||||
|
let selector = 0;
|
||||||
|
let hufGroup = groups[0];
|
||||||
|
let base = hufGroup.base;
|
||||||
|
let limit = hufGroup.limit;
|
||||||
|
|
||||||
|
// Main decoding loop
|
||||||
|
while (true) {
|
||||||
|
if (!symCountRemaining--) {
|
||||||
|
symCountRemaining = GROUP_SIZE - 1;
|
||||||
|
if (selector >= nSelectors) {
|
||||||
|
throwError('Invalid selector index: exceeds available groups', BZIP2_ERROR_CODES.INVALID_SELECTOR);
|
||||||
|
}
|
||||||
|
hufGroup = groups[this.selectors[selector++]];
|
||||||
|
base = hufGroup.base;
|
||||||
|
limit = hufGroup.limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
let i = hufGroup.minLen;
|
||||||
|
let j = bits(i) as number;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
if (i > hufGroup.maxLen) {
|
||||||
|
throwError('Huffman decoding error: bit length exceeds maximum allowed', BZIP2_ERROR_CODES.INVALID_HUFFMAN);
|
||||||
|
}
|
||||||
|
if (j <= limit[i]) break;
|
||||||
|
i++;
|
||||||
|
j = (j << 1) | (bits(1) as number);
|
||||||
|
}
|
||||||
|
|
||||||
|
j -= base[i];
|
||||||
|
if (j < 0 || j >= MAX_SYMBOLS) {
|
||||||
|
throwError('Symbol index out of bounds during Huffman decoding', BZIP2_ERROR_CODES.INVALID_HUFFMAN);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextSym = hufGroup.permute[j];
|
||||||
|
|
||||||
|
if (nextSym === SYMBOL_RUNA || nextSym === SYMBOL_RUNB) {
|
||||||
|
if (!runPos) {
|
||||||
|
runPos = 1;
|
||||||
|
j = 0;
|
||||||
|
}
|
||||||
|
if (nextSym === SYMBOL_RUNA) j += runPos;
|
||||||
|
else j += 2 * runPos;
|
||||||
|
runPos <<= 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runPos) {
|
||||||
|
runPos = 0;
|
||||||
|
const runLength = j;
|
||||||
|
if (count + runLength > bufsize) {
|
||||||
|
throwError('Run-length overflow: decoded run exceeds buffer capacity', BZIP2_ERROR_CODES.BUFFER_OVERFLOW);
|
||||||
|
}
|
||||||
|
const uc = this.symToByte[this.mtfSymbol[0]];
|
||||||
|
this.byteCount[uc] += runLength;
|
||||||
|
for (let t = 0; t < runLength; t++) {
|
||||||
|
buf[count++] = uc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nextSym > symTotal) break;
|
||||||
|
|
||||||
|
if (count >= bufsize) {
|
||||||
|
throwError('Buffer overflow: decoded data exceeds buffer capacity', BZIP2_ERROR_CODES.BUFFER_OVERFLOW);
|
||||||
|
}
|
||||||
|
|
||||||
|
const mtfIndex = nextSym - 1;
|
||||||
|
const uc = this.mtfSymbol[mtfIndex];
|
||||||
|
for (let k = mtfIndex - 1; k >= 0; k--) {
|
||||||
|
this.mtfSymbol[k + 1] = this.mtfSymbol[k];
|
||||||
|
}
|
||||||
|
this.mtfSymbol[0] = uc;
|
||||||
|
const decodedByte = this.symToByte[uc];
|
||||||
|
this.byteCount[decodedByte]++;
|
||||||
|
buf[count++] = decodedByte;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (origPtr < 0 || origPtr >= count) {
|
||||||
|
throwError('Invalid original pointer: position outside decoded block', BZIP2_ERROR_CODES.INVALID_POSITION);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inverse BWT transform
|
||||||
|
let j = 0;
|
||||||
|
for (let i = 0; i < 256; i++) {
|
||||||
|
const k = j + this.byteCount[i];
|
||||||
|
this.byteCount[i] = j;
|
||||||
|
j = k;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const uc = buf[i] & 0xff;
|
||||||
|
buf[this.byteCount[uc]] |= i << 8;
|
||||||
|
this.byteCount[uc]++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output decoded data
|
||||||
|
let pos = 0;
|
||||||
|
let current = 0;
|
||||||
|
let run = 0;
|
||||||
|
|
||||||
if (count) {
|
if (count) {
|
||||||
pos = buf[origPtr];
|
pos = buf[origPtr];
|
||||||
current = (pos & 0xff);
|
current = pos & 0xff;
|
||||||
pos >>= 8;
|
pos >>= 8;
|
||||||
run = -1;
|
run = -1;
|
||||||
}
|
|
||||||
count = count;
|
|
||||||
var copies, previous, outbyte;
|
|
||||||
while(count) {
|
|
||||||
count--;
|
|
||||||
previous = current;
|
|
||||||
pos = buf[pos];
|
|
||||||
current = pos & 0xff;
|
|
||||||
pos >>= 8;
|
|
||||||
if (run++ == 3) {
|
|
||||||
copies = current;
|
|
||||||
outbyte = previous;
|
|
||||||
current = -1;
|
|
||||||
} else {
|
|
||||||
copies = 1;
|
|
||||||
outbyte = current;
|
|
||||||
}
|
|
||||||
while(copies--) {
|
|
||||||
crc = ((crc << 8) ^ this.crcTable[((crc>>24) ^ outbyte) & 0xFF])&0xFFFFFFFF; // crc32
|
|
||||||
stream(outbyte);
|
|
||||||
}
|
|
||||||
if (current != previous) run = 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
crc = (crc ^ (-1)) >>> 0;
|
let remaining = count;
|
||||||
if ((crc|0) != (crcblock|0)) messageArg.Error("Error in bzip2: crc32 do not match");
|
while (remaining) {
|
||||||
streamCRC = (crc ^ ((streamCRC << 1) | (streamCRC >>> 31))) & 0xFFFFFFFF;
|
remaining--;
|
||||||
return streamCRC;
|
const previous = current;
|
||||||
};
|
pos = buf[pos];
|
||||||
};
|
current = pos & 0xff;
|
||||||
|
pos >>= 8;
|
||||||
|
|
||||||
|
let copies: number;
|
||||||
|
let outbyte: number;
|
||||||
|
|
||||||
|
if (run++ === 3) {
|
||||||
|
copies = current;
|
||||||
|
outbyte = previous;
|
||||||
|
current = -1;
|
||||||
|
} else {
|
||||||
|
copies = 1;
|
||||||
|
outbyte = current;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (copies--) {
|
||||||
|
crc = ((crc << 8) ^ this.crcTable[((crc >> 24) ^ outbyte) & 0xff]) & 0xffffffff;
|
||||||
|
stream(outbyte);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current !== previous) run = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
crc = (crc ^ -1) >>> 0;
|
||||||
|
if ((crc | 0) !== (crcblock | 0)) {
|
||||||
|
throwError('CRC32 mismatch: block checksum verification failed', BZIP2_ERROR_CODES.CRC_MISMATCH);
|
||||||
|
}
|
||||||
|
|
||||||
|
const newStreamCRC = (crc ^ (((streamCRC || 0) << 1) | ((streamCRC || 0) >>> 31))) & 0xffffffff;
|
||||||
|
return newStreamCRC;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,45 +1,53 @@
|
|||||||
import * as plugins from '../plugins.js';
|
import * as plugins from '../plugins.js';
|
||||||
|
import { Bzip2Error, BZIP2_ERROR_CODES } from '../errors.js';
|
||||||
|
import type { IBitReader } from '../interfaces.js';
|
||||||
|
|
||||||
import { Bzip2 } from './bzip2.js';
|
import { Bzip2 } from './bzip2.js';
|
||||||
import { bitIterator } from './bititerator.js';
|
import { bitIterator } from './bititerator.js';
|
||||||
|
|
||||||
export function unbzip2Stream() {
|
/**
|
||||||
|
* Creates a streaming BZIP2 decompression transform
|
||||||
|
*/
|
||||||
|
export function unbzip2Stream(): plugins.smartstream.SmartDuplex<Buffer, Buffer> {
|
||||||
const bzip2Instance = new Bzip2();
|
const bzip2Instance = new Bzip2();
|
||||||
var bufferQueue = [];
|
const bufferQueue: Buffer[] = [];
|
||||||
var hasBytes = 0;
|
let hasBytes = 0;
|
||||||
var blockSize = 0;
|
let blockSize = 0;
|
||||||
var broken = false;
|
let broken = false;
|
||||||
var done = false;
|
let bitReader: IBitReader | null = null;
|
||||||
var bitReader = null;
|
let streamCRC: number | null = null;
|
||||||
var streamCRC = null;
|
|
||||||
|
|
||||||
function decompressBlock() {
|
function decompressBlock(): Buffer | undefined {
|
||||||
if (!blockSize) {
|
if (!blockSize) {
|
||||||
blockSize = bzip2Instance.header(bitReader);
|
blockSize = bzip2Instance.header(bitReader!);
|
||||||
streamCRC = 0;
|
streamCRC = 0;
|
||||||
} else {
|
return undefined;
|
||||||
var bufsize = 100000 * blockSize;
|
|
||||||
var buf = new Int32Array(bufsize);
|
|
||||||
|
|
||||||
var chunk = [];
|
|
||||||
var f = function (b) {
|
|
||||||
chunk.push(b);
|
|
||||||
};
|
|
||||||
|
|
||||||
streamCRC = bzip2Instance.decompress(bitReader, f, buf, bufsize, streamCRC);
|
|
||||||
if (streamCRC === null) {
|
|
||||||
// reset for next bzip2 header
|
|
||||||
blockSize = 0;
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
return Buffer.from(chunk);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const bufsize = 100000 * blockSize;
|
||||||
|
const buf = new Int32Array(bufsize);
|
||||||
|
const chunk: number[] = [];
|
||||||
|
|
||||||
|
const outputFunc = (b: number): void => {
|
||||||
|
chunk.push(b);
|
||||||
|
};
|
||||||
|
|
||||||
|
streamCRC = bzip2Instance.decompress(bitReader!, outputFunc, buf, bufsize, streamCRC);
|
||||||
|
|
||||||
|
if (streamCRC === null) {
|
||||||
|
// Reset for next bzip2 header
|
||||||
|
blockSize = 0;
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Buffer.from(chunk);
|
||||||
}
|
}
|
||||||
|
|
||||||
var outlength = 0;
|
let outlength = 0;
|
||||||
const decompressAndPush = async () => {
|
|
||||||
if (broken) return;
|
const decompressAndPush = async (): Promise<Buffer | undefined> => {
|
||||||
|
if (broken) return undefined;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const resultChunk = decompressBlock();
|
const resultChunk = decompressBlock();
|
||||||
if (resultChunk) {
|
if (resultChunk) {
|
||||||
@@ -47,37 +55,39 @@ export function unbzip2Stream() {
|
|||||||
}
|
}
|
||||||
return resultChunk;
|
return resultChunk;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(e);
|
|
||||||
broken = true;
|
broken = true;
|
||||||
|
if (e instanceof Error) {
|
||||||
|
throw new Bzip2Error(`Decompression failed: ${e.message}`, BZIP2_ERROR_CODES.INVALID_BLOCK_DATA);
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let counter = 0;
|
|
||||||
return new plugins.smartstream.SmartDuplex({
|
return new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
||||||
objectMode: true,
|
objectMode: true,
|
||||||
name: 'bzip2',
|
name: 'bzip2',
|
||||||
debug: false,
|
|
||||||
highWaterMark: 1,
|
highWaterMark: 1,
|
||||||
writeFunction: async function (data, streamTools) {
|
writeFunction: async function (data, streamTools) {
|
||||||
// console.log(`got chunk ${counter++}`)
|
|
||||||
bufferQueue.push(data);
|
bufferQueue.push(data);
|
||||||
hasBytes += data.length;
|
hasBytes += data.length;
|
||||||
|
|
||||||
if (bitReader === null) {
|
if (bitReader === null) {
|
||||||
bitReader = bitIterator(function () {
|
bitReader = bitIterator(function () {
|
||||||
return bufferQueue.shift();
|
return bufferQueue.shift()!;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
|
|
||||||
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
|
const threshold = 25000 + 100000 * blockSize || 4;
|
||||||
|
while (!broken && hasBytes - bitReader.bytesRead + 1 >= threshold) {
|
||||||
const result = await decompressAndPush();
|
const result = await decompressAndPush();
|
||||||
if (!result) {
|
if (!result) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// console.log(result.toString());
|
|
||||||
await streamTools.push(result);
|
await streamTools.push(result);
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
},
|
},
|
||||||
finalFunction: async function (streamTools) {
|
finalFunction: async function (streamTools) {
|
||||||
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
|
|
||||||
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
||||||
const result = await decompressAndPush();
|
const result = await decompressAndPush();
|
||||||
if (!result) {
|
if (!result) {
|
||||||
@@ -85,9 +95,11 @@ export function unbzip2Stream() {
|
|||||||
}
|
}
|
||||||
await streamTools.push(result);
|
await streamTools.push(result);
|
||||||
}
|
}
|
||||||
if (!broken) {
|
|
||||||
if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
|
if (!broken && streamCRC !== null) {
|
||||||
|
this.emit('error', new Bzip2Error('Input stream ended prematurely', BZIP2_ERROR_CODES.PREMATURE_END));
|
||||||
}
|
}
|
||||||
|
return null;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,21 +1,41 @@
|
|||||||
import type { SmartArchive } from './classes.smartarchive.js';
|
import type { SmartArchive } from './classes.smartarchive.js';
|
||||||
|
import type { TSupportedMime } from './interfaces.js';
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Type for decompression streams
|
||||||
|
*/
|
||||||
|
export type TDecompressionStream =
|
||||||
|
| plugins.stream.Transform
|
||||||
|
| plugins.stream.Duplex
|
||||||
|
| plugins.tarStream.Extract;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Result of archive analysis
|
||||||
|
*/
|
||||||
export interface IAnalyzedResult {
|
export interface IAnalyzedResult {
|
||||||
fileType: plugins.fileType.FileTypeResult;
|
fileType: plugins.fileType.FileTypeResult | undefined;
|
||||||
isArchive: boolean;
|
isArchive: boolean;
|
||||||
resultStream: plugins.smartstream.SmartDuplex;
|
resultStream: plugins.smartstream.SmartDuplex<Buffer, Buffer>;
|
||||||
decompressionStream: plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract;
|
decompressionStream: TDecompressionStream;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyzes archive streams to detect format and provide decompression
|
||||||
|
*/
|
||||||
export class ArchiveAnalyzer {
|
export class ArchiveAnalyzer {
|
||||||
smartArchiveRef: SmartArchive;
|
private smartArchiveRef: SmartArchive;
|
||||||
|
|
||||||
constructor(smartArchiveRefArg: SmartArchive) {
|
constructor(smartArchiveRefArg: SmartArchive) {
|
||||||
this.smartArchiveRef = smartArchiveRefArg;
|
this.smartArchiveRef = smartArchiveRefArg;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async mimeTypeIsArchive(mimeType: string): Promise<boolean> {
|
/**
|
||||||
|
* Check if a MIME type represents an archive format
|
||||||
|
*/
|
||||||
|
private async mimeTypeIsArchive(mimeType: string | undefined): Promise<boolean> {
|
||||||
|
if (!mimeType) return false;
|
||||||
|
|
||||||
const archiveMimeTypes: Set<string> = new Set([
|
const archiveMimeTypes: Set<string> = new Set([
|
||||||
'application/zip',
|
'application/zip',
|
||||||
'application/x-rar-compressed',
|
'application/x-rar-compressed',
|
||||||
@@ -23,44 +43,46 @@ export class ArchiveAnalyzer {
|
|||||||
'application/gzip',
|
'application/gzip',
|
||||||
'application/x-7z-compressed',
|
'application/x-7z-compressed',
|
||||||
'application/x-bzip2',
|
'application/x-bzip2',
|
||||||
// Add other archive mime types here
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return archiveMimeTypes.has(mimeType);
|
return archiveMimeTypes.has(mimeType);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
private async getDecompressionStream(
|
* Get the appropriate decompression stream for a MIME type
|
||||||
mimeTypeArg: plugins.fileType.FileTypeResult['mime']
|
*/
|
||||||
): Promise<plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract> {
|
private async getDecompressionStream(mimeTypeArg: TSupportedMime): Promise<TDecompressionStream> {
|
||||||
switch (mimeTypeArg) {
|
switch (mimeTypeArg) {
|
||||||
case 'application/gzip':
|
case 'application/gzip':
|
||||||
return this.smartArchiveRef.gzipTools.getDecompressionStream();
|
return this.smartArchiveRef.gzipTools.getDecompressionStream();
|
||||||
case 'application/zip':
|
case 'application/zip':
|
||||||
return this.smartArchiveRef.zipTools.getDecompressionStream();
|
return this.smartArchiveRef.zipTools.getDecompressionStream();
|
||||||
case 'application/x-bzip2':
|
case 'application/x-bzip2':
|
||||||
return await this.smartArchiveRef.bzip2Tools.getDecompressionStream(); // replace with your own bzip2 decompression stream
|
return this.smartArchiveRef.bzip2Tools.getDecompressionStream();
|
||||||
case 'application/x-tar':
|
case 'application/x-tar':
|
||||||
return this.smartArchiveRef.tarTools.getDecompressionStream(); // replace with your own tar decompression stream
|
return this.smartArchiveRef.tarTools.getDecompressionStream();
|
||||||
default:
|
default:
|
||||||
// Handle unsupported formats or no decompression needed
|
// Handle unsupported formats or no decompression needed
|
||||||
return plugins.smartstream.createPassThrough();
|
return plugins.smartstream.createPassThrough();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public getAnalyzedStream() {
|
/**
|
||||||
|
* Create an analyzed stream that detects archive type and provides decompression
|
||||||
|
* Emits a single IAnalyzedResult object
|
||||||
|
*/
|
||||||
|
public getAnalyzedStream(): plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult> {
|
||||||
let firstRun = true;
|
let firstRun = true;
|
||||||
const resultStream = plugins.smartstream.createPassThrough();
|
const resultStream = plugins.smartstream.createPassThrough();
|
||||||
|
|
||||||
const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
|
const analyzerstream = new plugins.smartstream.SmartDuplex<Buffer, IAnalyzedResult>({
|
||||||
readableObjectMode: true,
|
readableObjectMode: true,
|
||||||
writeFunction: async (chunkArg: Buffer, streamtools) => {
|
writeFunction: async (chunkArg: Buffer, streamtools) => {
|
||||||
if (firstRun) {
|
if (firstRun) {
|
||||||
firstRun = false;
|
firstRun = false;
|
||||||
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
|
const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg);
|
||||||
const decompressionStream = await this.getDecompressionStream(fileType?.mime as any);
|
const decompressionStream = await this.getDecompressionStream(fileType?.mime as TSupportedMime);
|
||||||
/**
|
|
||||||
* analyzed stream emits once with this object
|
|
||||||
*/
|
|
||||||
const result: IAnalyzedResult = {
|
const result: IAnalyzedResult = {
|
||||||
fileType,
|
fileType,
|
||||||
isArchive: await this.mimeTypeIsArchive(fileType?.mime),
|
isArchive: await this.mimeTypeIsArchive(fileType?.mime),
|
||||||
@@ -72,11 +94,12 @@ export class ArchiveAnalyzer {
|
|||||||
await resultStream.backpressuredPush(chunkArg);
|
await resultStream.backpressuredPush(chunkArg);
|
||||||
return null;
|
return null;
|
||||||
},
|
},
|
||||||
finalFunction: async (tools) => {
|
finalFunction: async () => {
|
||||||
resultStream.push(null);
|
resultStream.push(null);
|
||||||
return null;
|
return null;
|
||||||
}
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return analyzerstream;
|
return analyzerstream;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,56 +1,138 @@
|
|||||||
import type { SmartArchive } from './classes.smartarchive.js';
|
import * as plugins from './plugins.js';
|
||||||
import * as plugins from './plugins.js'
|
import type { TCompressionLevel } from './interfaces.js';
|
||||||
|
|
||||||
// This class wraps fflate's gunzip in a Node.js Transform stream
|
/**
|
||||||
export class CompressGunzipTransform extends plugins.stream.Transform {
|
* Transform stream for GZIP compression using fflate
|
||||||
constructor() {
|
*/
|
||||||
|
export class GzipCompressionTransform extends plugins.stream.Transform {
|
||||||
|
private gzip: plugins.fflate.Gzip;
|
||||||
|
|
||||||
|
constructor(level: TCompressionLevel = 6) {
|
||||||
super();
|
super();
|
||||||
}
|
|
||||||
|
|
||||||
_transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) {
|
// Create a streaming Gzip compressor
|
||||||
plugins.fflate.gunzip(chunk, (err, decompressed) => {
|
this.gzip = new plugins.fflate.Gzip({ level }, (chunk, final) => {
|
||||||
if (err) {
|
this.push(Buffer.from(chunk));
|
||||||
callback(err);
|
if (final) {
|
||||||
} else {
|
this.push(null);
|
||||||
this.push(decompressed);
|
|
||||||
callback();
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// DecompressGunzipTransform class that extends the Node.js Transform stream to
|
_transform(
|
||||||
// create a stream that decompresses GZip-compressed data using fflate's gunzip function
|
chunk: Buffer,
|
||||||
export class DecompressGunzipTransform extends plugins.stream.Transform {
|
encoding: BufferEncoding,
|
||||||
constructor() {
|
callback: plugins.stream.TransformCallback
|
||||||
super();
|
): void {
|
||||||
|
try {
|
||||||
|
this.gzip.push(chunk, false);
|
||||||
|
callback();
|
||||||
|
} catch (err) {
|
||||||
|
callback(err as Error);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
_transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) {
|
_flush(callback: plugins.stream.TransformCallback): void {
|
||||||
// Use fflate's gunzip function to decompress the chunk
|
try {
|
||||||
plugins.fflate.gunzip(chunk, (err, decompressed) => {
|
this.gzip.push(new Uint8Array(0), true);
|
||||||
if (err) {
|
callback();
|
||||||
// If an error occurs during decompression, pass the error to the callback
|
} catch (err) {
|
||||||
callback(err);
|
callback(err as Error);
|
||||||
} else {
|
}
|
||||||
// If decompression is successful, push the decompressed data into the stream
|
}
|
||||||
this.push(decompressed);
|
}
|
||||||
callback();
|
|
||||||
|
/**
|
||||||
|
* Transform stream for GZIP decompression using fflate
|
||||||
|
*/
|
||||||
|
export class GzipDecompressionTransform extends plugins.stream.Transform {
|
||||||
|
private gunzip: plugins.fflate.Gunzip;
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super();
|
||||||
|
|
||||||
|
// Create a streaming Gunzip decompressor
|
||||||
|
this.gunzip = new plugins.fflate.Gunzip((chunk, final) => {
|
||||||
|
this.push(Buffer.from(chunk));
|
||||||
|
if (final) {
|
||||||
|
this.push(null);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_transform(
|
||||||
|
chunk: Buffer,
|
||||||
|
encoding: BufferEncoding,
|
||||||
|
callback: plugins.stream.TransformCallback
|
||||||
|
): void {
|
||||||
|
try {
|
||||||
|
this.gunzip.push(chunk, false);
|
||||||
|
callback();
|
||||||
|
} catch (err) {
|
||||||
|
callback(err as Error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_flush(callback: plugins.stream.TransformCallback): void {
|
||||||
|
try {
|
||||||
|
this.gunzip.push(new Uint8Array(0), true);
|
||||||
|
callback();
|
||||||
|
} catch (err) {
|
||||||
|
callback(err as Error);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GZIP compression and decompression utilities
|
||||||
|
*/
|
||||||
export class GzipTools {
|
export class GzipTools {
|
||||||
constructor() {
|
/**
|
||||||
|
* Get a streaming compression transform
|
||||||
|
*/
|
||||||
|
public getCompressionStream(level?: TCompressionLevel): plugins.stream.Transform {
|
||||||
|
return new GzipCompressionTransform(level);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getCompressionStream() {
|
/**
|
||||||
return new CompressGunzipTransform();
|
* Get a streaming decompression transform
|
||||||
|
*/
|
||||||
|
public getDecompressionStream(): plugins.stream.Transform {
|
||||||
|
return new GzipDecompressionTransform();
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDecompressionStream() {
|
/**
|
||||||
return new DecompressGunzipTransform();
|
* Compress data synchronously
|
||||||
|
*/
|
||||||
|
public compressSync(data: Buffer, level?: TCompressionLevel): Buffer {
|
||||||
|
const options = level !== undefined ? { level } : undefined;
|
||||||
|
return Buffer.from(plugins.fflate.gzipSync(data, options));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decompress data synchronously
|
||||||
|
*/
|
||||||
|
public decompressSync(data: Buffer): Buffer {
|
||||||
|
return Buffer.from(plugins.fflate.gunzipSync(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compress data asynchronously
|
||||||
|
* Note: Uses sync version for Deno compatibility (fflate async uses Web Workers
|
||||||
|
* which have issues in Deno)
|
||||||
|
*/
|
||||||
|
public async compress(data: Buffer, level?: TCompressionLevel): Promise<Buffer> {
|
||||||
|
// Use sync version wrapped in Promise for cross-runtime compatibility
|
||||||
|
return this.compressSync(data, level);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decompress data asynchronously
|
||||||
|
* Note: Uses sync version for Deno compatibility (fflate async uses Web Workers
|
||||||
|
* which have issues in Deno)
|
||||||
|
*/
|
||||||
|
public async decompress(data: Buffer): Promise<Buffer> {
|
||||||
|
// Use sync version wrapped in Promise for cross-runtime compatibility
|
||||||
|
return this.decompressSync(data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,157 +1,419 @@
|
|||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import * as paths from './paths.js';
|
import type {
|
||||||
|
IArchiveEntry,
|
||||||
|
IArchiveEntryInfo,
|
||||||
|
IArchiveInfo,
|
||||||
|
TArchiveFormat,
|
||||||
|
TCompressionLevel,
|
||||||
|
TEntryFilter,
|
||||||
|
} from './interfaces.js';
|
||||||
|
|
||||||
import { Bzip2Tools } from './classes.bzip2tools.js';
|
import { Bzip2Tools } from './classes.bzip2tools.js';
|
||||||
import { GzipTools } from './classes.gziptools.js';
|
import { GzipTools } from './classes.gziptools.js';
|
||||||
import { TarTools } from './classes.tartools.js';
|
import { TarTools } from './classes.tartools.js';
|
||||||
import { ZipTools } from './classes.ziptools.js';
|
import { ZipTools } from './classes.ziptools.js';
|
||||||
|
|
||||||
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
||||||
|
|
||||||
import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js';
|
/**
|
||||||
|
* Pending directory entry for async resolution
|
||||||
|
*/
|
||||||
|
interface IPendingDirectory {
|
||||||
|
sourcePath: string;
|
||||||
|
archiveBase?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Main class for archive manipulation with fluent API
|
||||||
|
* Supports TAR, ZIP, GZIP, and BZIP2 formats
|
||||||
|
*
|
||||||
|
* @example Extraction from URL
|
||||||
|
* ```typescript
|
||||||
|
* await SmartArchive.create()
|
||||||
|
* .url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||||
|
* .stripComponents(1)
|
||||||
|
* .extract('./node_modules/lodash');
|
||||||
|
* ```
|
||||||
|
*
|
||||||
|
* @example Creation with thenable
|
||||||
|
* ```typescript
|
||||||
|
* const archive = await SmartArchive.create()
|
||||||
|
* .format('tar.gz')
|
||||||
|
* .compression(9)
|
||||||
|
* .entry('config.json', JSON.stringify(config))
|
||||||
|
* .directory('./src');
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
export class SmartArchive {
|
export class SmartArchive {
|
||||||
// STATIC
|
// ============================================
|
||||||
public static async fromArchiveUrl(urlArg: string): Promise<SmartArchive> {
|
// STATIC ENTRY POINT
|
||||||
const smartArchiveInstance = new SmartArchive();
|
// ============================================
|
||||||
smartArchiveInstance.sourceUrl = urlArg;
|
|
||||||
return smartArchiveInstance;
|
/**
|
||||||
|
* Create a new SmartArchive instance for fluent configuration
|
||||||
|
*/
|
||||||
|
public static create(): SmartArchive {
|
||||||
|
return new SmartArchive();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async fromArchiveFile(filePathArg: string): Promise<SmartArchive> {
|
// ============================================
|
||||||
const smartArchiveInstance = new SmartArchive();
|
// TOOLS (public for internal use)
|
||||||
smartArchiveInstance.sourceFilePath = filePathArg;
|
// ============================================
|
||||||
return smartArchiveInstance;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static async fromArchiveStream(
|
|
||||||
streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform
|
|
||||||
): Promise<SmartArchive> {
|
|
||||||
const smartArchiveInstance = new SmartArchive();
|
|
||||||
smartArchiveInstance.sourceStream = streamArg;
|
|
||||||
return smartArchiveInstance;
|
|
||||||
}
|
|
||||||
|
|
||||||
// INSTANCE
|
|
||||||
public tarTools = new TarTools();
|
public tarTools = new TarTools();
|
||||||
public zipTools = new ZipTools();
|
public zipTools = new ZipTools();
|
||||||
public gzipTools = new GzipTools();
|
public gzipTools = new GzipTools();
|
||||||
public bzip2Tools = new Bzip2Tools(this);
|
public bzip2Tools = new Bzip2Tools(this);
|
||||||
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
||||||
|
|
||||||
public sourceUrl: string;
|
// ============================================
|
||||||
public sourceFilePath: string;
|
// SOURCE STATE (extraction mode)
|
||||||
public sourceStream: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
|
// ============================================
|
||||||
|
|
||||||
public archiveName: string;
|
private sourceUrl?: string;
|
||||||
public singleFileMode: boolean = false;
|
private sourceFilePath?: string;
|
||||||
|
private sourceStream?: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
|
||||||
|
|
||||||
public addedDirectories: string[] = [];
|
// ============================================
|
||||||
public addedFiles: (plugins.smartfile.SmartFile | plugins.smartfile.StreamFile)[] = [];
|
// CREATION STATE
|
||||||
public addedUrls: string[] = [];
|
// ============================================
|
||||||
|
|
||||||
|
private archiveBuffer?: Buffer;
|
||||||
|
private creationFormat?: TArchiveFormat;
|
||||||
|
private _compressionLevel: TCompressionLevel = 6;
|
||||||
|
private pendingEntries: IArchiveEntry[] = [];
|
||||||
|
private pendingDirectories: IPendingDirectory[] = [];
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// FLUENT STATE
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
private _mode: 'extract' | 'create' | null = null;
|
||||||
|
private _filters: TEntryFilter[] = [];
|
||||||
|
private _excludePatterns: RegExp[] = [];
|
||||||
|
private _includePatterns: RegExp[] = [];
|
||||||
|
private _stripComponents: number = 0;
|
||||||
|
private _overwrite: boolean = false;
|
||||||
|
private _fileName?: string;
|
||||||
|
|
||||||
constructor() {}
|
constructor() {}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// SOURCE METHODS (set extraction mode)
|
||||||
|
// ============================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gets the original archive stream
|
* Load archive from URL
|
||||||
*/
|
*/
|
||||||
public async getArchiveStream() {
|
public url(urlArg: string): this {
|
||||||
if (this.sourceStream) {
|
this.ensureNotInCreateMode('url');
|
||||||
return this.sourceStream;
|
this._mode = 'extract';
|
||||||
}
|
this.sourceUrl = urlArg;
|
||||||
if (this.sourceUrl) {
|
return this;
|
||||||
const urlStream = await plugins.smartrequest.getStream(this.sourceUrl);
|
|
||||||
return urlStream;
|
|
||||||
}
|
|
||||||
if (this.sourceFilePath) {
|
|
||||||
const fileStream = plugins.smartfile.fs.toReadStream(this.sourceFilePath);
|
|
||||||
return fileStream;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async exportToTarGzStream() {
|
/**
|
||||||
const tarPackStream = await this.tarTools.getPackStream();
|
* Load archive from file path
|
||||||
const gzipStream = await this.gzipTools.getCompressionStream();
|
*/
|
||||||
// const archiveStream = tarPackStream.pipe(gzipStream);
|
public file(pathArg: string): this {
|
||||||
// return archiveStream;
|
this.ensureNotInCreateMode('file');
|
||||||
|
this._mode = 'extract';
|
||||||
|
this.sourceFilePath = pathArg;
|
||||||
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async exportToFs(targetDir: string, fileNameArg?: string): Promise<void> {
|
/**
|
||||||
|
* Load archive from readable stream
|
||||||
|
*/
|
||||||
|
public stream(streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform): this {
|
||||||
|
this.ensureNotInCreateMode('stream');
|
||||||
|
this._mode = 'extract';
|
||||||
|
this.sourceStream = streamArg;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load archive from buffer
|
||||||
|
*/
|
||||||
|
public buffer(bufferArg: Buffer): this {
|
||||||
|
this.ensureNotInCreateMode('buffer');
|
||||||
|
this._mode = 'extract';
|
||||||
|
this.sourceStream = plugins.stream.Readable.from(bufferArg);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// FORMAT METHODS (set creation mode)
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set output format for archive creation
|
||||||
|
*/
|
||||||
|
public format(fmt: TArchiveFormat): this {
|
||||||
|
this.ensureNotInExtractMode('format');
|
||||||
|
this._mode = 'create';
|
||||||
|
this.creationFormat = fmt;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set compression level (0-9)
|
||||||
|
*/
|
||||||
|
public compression(level: TCompressionLevel): this {
|
||||||
|
this._compressionLevel = level;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// CONTENT METHODS (creation mode)
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a single file entry to the archive
|
||||||
|
*/
|
||||||
|
public entry(archivePath: string, content: string | Buffer): this {
|
||||||
|
this.ensureNotInExtractMode('entry');
|
||||||
|
if (!this._mode) this._mode = 'create';
|
||||||
|
this.pendingEntries.push({ archivePath, content });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add multiple entries to the archive
|
||||||
|
*/
|
||||||
|
public entries(entriesArg: Array<{ archivePath: string; content: string | Buffer }>): this {
|
||||||
|
this.ensureNotInExtractMode('entries');
|
||||||
|
if (!this._mode) this._mode = 'create';
|
||||||
|
for (const e of entriesArg) {
|
||||||
|
this.pendingEntries.push({ archivePath: e.archivePath, content: e.content });
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add an entire directory to the archive (queued, resolved at build time)
|
||||||
|
*/
|
||||||
|
public directory(sourcePath: string, archiveBase?: string): this {
|
||||||
|
this.ensureNotInExtractMode('directory');
|
||||||
|
if (!this._mode) this._mode = 'create';
|
||||||
|
this.pendingDirectories.push({ sourcePath, archiveBase });
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a SmartFile to the archive
|
||||||
|
*/
|
||||||
|
public addSmartFile(fileArg: plugins.smartfile.SmartFile, archivePath?: string): this {
|
||||||
|
this.ensureNotInExtractMode('addSmartFile');
|
||||||
|
if (!this._mode) this._mode = 'create';
|
||||||
|
this.pendingEntries.push({
|
||||||
|
archivePath: archivePath || fileArg.relative,
|
||||||
|
content: fileArg,
|
||||||
|
});
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add a StreamFile to the archive
|
||||||
|
*/
|
||||||
|
public addStreamFile(fileArg: plugins.smartfile.StreamFile, archivePath?: string): this {
|
||||||
|
this.ensureNotInExtractMode('addStreamFile');
|
||||||
|
if (!this._mode) this._mode = 'create';
|
||||||
|
this.pendingEntries.push({
|
||||||
|
archivePath: archivePath || fileArg.relativeFilePath,
|
||||||
|
content: fileArg,
|
||||||
|
});
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// FILTER METHODS (both modes)
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Filter entries by predicate function
|
||||||
|
*/
|
||||||
|
public filter(predicate: TEntryFilter): this {
|
||||||
|
this._filters.push(predicate);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Include only entries matching the pattern
|
||||||
|
*/
|
||||||
|
public include(pattern: string | RegExp): this {
|
||||||
|
const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
|
||||||
|
this._includePatterns.push(regex);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Exclude entries matching the pattern
|
||||||
|
*/
|
||||||
|
public exclude(pattern: string | RegExp): this {
|
||||||
|
const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
|
||||||
|
this._excludePatterns.push(regex);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// EXTRACTION OPTIONS
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strip N leading path components from extracted files
|
||||||
|
*/
|
||||||
|
public stripComponents(n: number): this {
|
||||||
|
this._stripComponents = n;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Overwrite existing files during extraction
|
||||||
|
*/
|
||||||
|
public overwrite(value: boolean = true): this {
|
||||||
|
this._overwrite = value;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set output filename for single-file archives (gz, bz2)
|
||||||
|
*/
|
||||||
|
public fileName(name: string): this {
|
||||||
|
this._fileName = name;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// TERMINAL METHODS - EXTRACTION
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract archive to filesystem directory
|
||||||
|
*/
|
||||||
|
public async extract(targetDir: string): Promise<void> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
const done = plugins.smartpromise.defer<void>();
|
const done = plugins.smartpromise.defer<void>();
|
||||||
const streamFileStream = await this.exportToStreamOfStreamFiles();
|
const streamFileStream = await this.toStreamFiles();
|
||||||
|
|
||||||
streamFileStream.pipe(
|
streamFileStream.pipe(
|
||||||
new plugins.smartstream.SmartDuplex({
|
new plugins.smartstream.SmartDuplex({
|
||||||
objectMode: true,
|
objectMode: true,
|
||||||
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile) => {
|
||||||
const done = plugins.smartpromise.defer<void>();
|
const innerDone = plugins.smartpromise.defer<void>();
|
||||||
console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path');
|
|
||||||
const streamFile = streamFileArg;
|
const streamFile = streamFileArg;
|
||||||
|
let relativePath = streamFile.relativeFilePath || this._fileName || 'extracted_file';
|
||||||
|
|
||||||
|
// Apply stripComponents
|
||||||
|
if (this._stripComponents > 0) {
|
||||||
|
const parts = relativePath.split('/');
|
||||||
|
relativePath = parts.slice(this._stripComponents).join('/');
|
||||||
|
if (!relativePath) {
|
||||||
|
innerDone.resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply filter
|
||||||
|
const filterFn = this.buildFilterFunction();
|
||||||
|
if (filterFn) {
|
||||||
|
const entryInfo: IArchiveEntryInfo = {
|
||||||
|
path: relativePath,
|
||||||
|
size: 0,
|
||||||
|
isDirectory: false,
|
||||||
|
isFile: true,
|
||||||
|
};
|
||||||
|
if (!filterFn(entryInfo)) {
|
||||||
|
innerDone.resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const readStream = await streamFile.createReadStream();
|
const readStream = await streamFile.createReadStream();
|
||||||
await plugins.smartfile.fs.ensureDir(targetDir);
|
await plugins.fsPromises.mkdir(targetDir, { recursive: true });
|
||||||
const writePath = plugins.path.join(
|
const writePath = plugins.path.join(targetDir, relativePath);
|
||||||
targetDir,
|
await plugins.fsPromises.mkdir(plugins.path.dirname(writePath), { recursive: true });
|
||||||
streamFile.relativeFilePath || fileNameArg
|
const writeStream = plugins.fs.createWriteStream(writePath);
|
||||||
);
|
|
||||||
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
|
|
||||||
const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath);
|
|
||||||
readStream.pipe(writeStream);
|
readStream.pipe(writeStream);
|
||||||
writeStream.on('finish', () => {
|
writeStream.on('finish', () => {
|
||||||
done.resolve();
|
innerDone.resolve();
|
||||||
});
|
});
|
||||||
await done.promise;
|
await innerDone.promise;
|
||||||
},
|
},
|
||||||
finalFunction: async () => {
|
finalFunction: async () => {
|
||||||
done.resolve();
|
done.resolve();
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
return done.promise;
|
return done.promise;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async exportToStreamOfStreamFiles() {
|
/**
|
||||||
|
* Extract archive to a stream of StreamFile objects
|
||||||
|
*/
|
||||||
|
public async toStreamFiles(): Promise<plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
|
||||||
const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
|
const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
|
||||||
objectMode: true,
|
objectMode: true,
|
||||||
});
|
});
|
||||||
const archiveStream = await this.getArchiveStream();
|
|
||||||
|
// Guard to prevent multiple signalEnd calls
|
||||||
|
let hasSignaledEnd = false;
|
||||||
|
const safeSignalEnd = () => {
|
||||||
|
if (!hasSignaledEnd) {
|
||||||
|
hasSignaledEnd = true;
|
||||||
|
streamFileIntake.signalEnd();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const archiveStream = await this.getSourceStream();
|
||||||
const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();
|
const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();
|
||||||
|
|
||||||
// lets create a function that can be called multiple times to unpack layers of archives
|
|
||||||
const createUnpackStream = () =>
|
const createUnpackStream = () =>
|
||||||
plugins.smartstream.createTransformFunction<IAnalyzedResult, any>(
|
plugins.smartstream.createTransformFunction<IAnalyzedResult, void>(
|
||||||
async (analyzedResultChunk) => {
|
async (analyzedResultChunk) => {
|
||||||
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
|
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
|
||||||
const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
|
const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract;
|
||||||
|
|
||||||
tarStream.on('entry', async (header, stream, next) => {
|
tarStream.on('entry', async (header, stream, next) => {
|
||||||
if (header.type === 'directory') {
|
if (header.type === 'directory') {
|
||||||
console.log(`tar stream directory: ${header.name} ... skipping!`);
|
stream.resume();
|
||||||
next();
|
stream.on('end', () => next());
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
console.log(`tar stream file: ${header.name}`);
|
|
||||||
const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name);
|
const passThrough = new plugins.stream.PassThrough();
|
||||||
|
const streamfile = plugins.smartfile.StreamFile.fromStream(passThrough, header.name);
|
||||||
streamFileIntake.push(streamfile);
|
streamFileIntake.push(streamfile);
|
||||||
stream.on('end', function () {
|
stream.pipe(passThrough);
|
||||||
next(); // ready for next entry
|
stream.on('end', () => {
|
||||||
|
passThrough.end();
|
||||||
|
next();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
tarStream.on('finish', function () {
|
|
||||||
console.log('finished');
|
tarStream.on('finish', () => {
|
||||||
streamFileIntake.signalEnd();
|
safeSignalEnd();
|
||||||
});
|
});
|
||||||
|
|
||||||
analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
|
analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream);
|
||||||
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
|
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
|
||||||
analyzedResultChunk.resultStream
|
analyzedResultChunk.resultStream
|
||||||
.pipe(analyzedResultChunk.decompressionStream)
|
.pipe(analyzedResultChunk.decompressionStream)
|
||||||
.pipe(new plugins.smartstream.SmartDuplex({
|
.pipe(
|
||||||
objectMode: true,
|
new plugins.smartstream.SmartDuplex({
|
||||||
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => {
|
objectMode: true,
|
||||||
streamFileIntake.push(streamFileArg);
|
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile) => {
|
||||||
},
|
streamFileIntake.push(streamFileArg);
|
||||||
finalFunction: async () => {
|
},
|
||||||
streamFileIntake.signalEnd();
|
finalFunction: async () => {
|
||||||
}
|
safeSignalEnd();
|
||||||
}));
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
} else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
|
} else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) {
|
||||||
|
// For nested archives (like gzip containing tar)
|
||||||
analyzedResultChunk.resultStream
|
analyzedResultChunk.resultStream
|
||||||
.pipe(analyzedResultChunk.decompressionStream)
|
.pipe(analyzedResultChunk.decompressionStream)
|
||||||
.pipe(createAnalyzedStream())
|
.pipe(createAnalyzedStream())
|
||||||
@@ -162,15 +424,440 @@ export class SmartArchive {
|
|||||||
analyzedResultChunk.fileType?.ext
|
analyzedResultChunk.fileType?.ext
|
||||||
);
|
);
|
||||||
streamFileIntake.push(streamFile);
|
streamFileIntake.push(streamFile);
|
||||||
streamFileIntake.signalEnd();
|
safeSignalEnd();
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{ objectMode: true }
|
||||||
objectMode: true,
|
|
||||||
}
|
|
||||||
);
|
);
|
||||||
|
|
||||||
archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());
|
archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream());
|
||||||
return streamFileIntake;
|
return streamFileIntake;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract archive to an array of SmartFile objects (in-memory)
|
||||||
|
*/
|
||||||
|
public async toSmartFiles(): Promise<plugins.smartfile.SmartFile[]> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
const streamFiles = await this.toStreamFiles();
|
||||||
|
const smartFiles: plugins.smartfile.SmartFile[] = [];
|
||||||
|
const filterFn = this.buildFilterFunction();
|
||||||
|
const pendingConversions: Promise<void>[] = [];
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
streamFiles.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||||
|
// Track all async conversions to ensure they complete before resolving
|
||||||
|
const conversion = (async () => {
|
||||||
|
try {
|
||||||
|
const smartFile = await streamFile.toSmartFile();
|
||||||
|
|
||||||
|
// Apply filter if configured
|
||||||
|
if (filterFn) {
|
||||||
|
const passes = filterFn({
|
||||||
|
path: smartFile.relative,
|
||||||
|
size: smartFile.contents.length,
|
||||||
|
isDirectory: false,
|
||||||
|
isFile: true,
|
||||||
|
});
|
||||||
|
if (!passes) return;
|
||||||
|
}
|
||||||
|
|
||||||
|
smartFiles.push(smartFile);
|
||||||
|
} catch (err) {
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
pendingConversions.push(conversion);
|
||||||
|
});
|
||||||
|
streamFiles.on('end', async () => {
|
||||||
|
// Wait for all conversions to complete before resolving
|
||||||
|
await Promise.all(pendingConversions);
|
||||||
|
resolve(smartFiles);
|
||||||
|
});
|
||||||
|
streamFiles.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract a single file from the archive by path
|
||||||
|
*/
|
||||||
|
public async extractFile(filePath: string): Promise<plugins.smartfile.SmartFile | null> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
const streamFiles = await this.toStreamFiles();
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let found = false;
|
||||||
|
|
||||||
|
streamFiles.on('data', async (streamFile: plugins.smartfile.StreamFile) => {
|
||||||
|
if (streamFile.relativeFilePath === filePath || streamFile.relativeFilePath?.endsWith(filePath)) {
|
||||||
|
found = true;
|
||||||
|
try {
|
||||||
|
const smartFile = await streamFile.toSmartFile();
|
||||||
|
resolve(smartFile);
|
||||||
|
} catch (err) {
|
||||||
|
reject(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
streamFiles.on('end', () => {
|
||||||
|
if (!found) {
|
||||||
|
resolve(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
streamFiles.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// TERMINAL METHODS - OUTPUT
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build and finalize the archive, returning this instance
|
||||||
|
*/
|
||||||
|
public async build(): Promise<SmartArchive> {
|
||||||
|
await this.doBuild();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal build implementation (avoids thenable recursion)
|
||||||
|
*/
|
||||||
|
private async doBuild(): Promise<void> {
|
||||||
|
if (this._mode === 'extract') {
|
||||||
|
// For extraction mode, nothing to build
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.archiveBuffer) {
|
||||||
|
// Already built
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For creation mode, build the archive buffer
|
||||||
|
this.ensureCreationFormat();
|
||||||
|
await this.resolveDirectories();
|
||||||
|
|
||||||
|
const entries = this.getFilteredEntries();
|
||||||
|
|
||||||
|
if (this.creationFormat === 'tar' || this.creationFormat === 'tar.gz' || this.creationFormat === 'tgz') {
|
||||||
|
if (this.creationFormat === 'tar') {
|
||||||
|
this.archiveBuffer = await this.tarTools.packFiles(entries);
|
||||||
|
} else {
|
||||||
|
this.archiveBuffer = await this.tarTools.packFilesToTarGz(entries, this._compressionLevel);
|
||||||
|
}
|
||||||
|
} else if (this.creationFormat === 'zip') {
|
||||||
|
this.archiveBuffer = await this.zipTools.createZip(entries, this._compressionLevel);
|
||||||
|
} else if (this.creationFormat === 'gz') {
|
||||||
|
if (entries.length !== 1) {
|
||||||
|
throw new Error('GZIP format only supports a single file');
|
||||||
|
}
|
||||||
|
let content: Buffer;
|
||||||
|
if (typeof entries[0].content === 'string') {
|
||||||
|
content = Buffer.from(entries[0].content);
|
||||||
|
} else if (Buffer.isBuffer(entries[0].content)) {
|
||||||
|
content = entries[0].content;
|
||||||
|
} else {
|
||||||
|
throw new Error('GZIP format requires string or Buffer content');
|
||||||
|
}
|
||||||
|
this.archiveBuffer = await this.gzipTools.compress(content, this._compressionLevel);
|
||||||
|
} else {
|
||||||
|
throw new Error(`Unsupported format: ${this.creationFormat}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build archive and return as Buffer
|
||||||
|
*/
|
||||||
|
public async toBuffer(): Promise<Buffer> {
|
||||||
|
if (this._mode === 'create' && !this.archiveBuffer) {
|
||||||
|
await this.doBuild();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.archiveBuffer) {
|
||||||
|
return this.archiveBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For extraction mode, get the source as buffer
|
||||||
|
const stream = await this.getSourceStream();
|
||||||
|
return this.streamToBuffer(stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build archive and write to file
|
||||||
|
*/
|
||||||
|
public async toFile(filePath: string): Promise<void> {
|
||||||
|
const buffer = await this.toBuffer();
|
||||||
|
await plugins.fsPromises.mkdir(plugins.path.dirname(filePath), { recursive: true });
|
||||||
|
await plugins.fsPromises.writeFile(filePath, buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get archive as a readable stream
|
||||||
|
*/
|
||||||
|
public async toStream(): Promise<plugins.stream.Readable> {
|
||||||
|
if (this._mode === 'create' && !this.archiveBuffer) {
|
||||||
|
await this.doBuild();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.archiveBuffer) {
|
||||||
|
return plugins.stream.Readable.from(this.archiveBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.getSourceStream();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// TERMINAL METHODS - ANALYSIS
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Analyze the archive and return metadata
|
||||||
|
*/
|
||||||
|
public async analyze(): Promise<IArchiveInfo> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
const stream = await this.getSourceStream();
|
||||||
|
const firstChunk = await this.readFirstChunk(stream);
|
||||||
|
const fileType = await plugins.fileType.fileTypeFromBuffer(firstChunk);
|
||||||
|
|
||||||
|
let format: TArchiveFormat | null = null;
|
||||||
|
let isCompressed = false;
|
||||||
|
let isArchive = false;
|
||||||
|
|
||||||
|
if (fileType) {
|
||||||
|
switch (fileType.mime) {
|
||||||
|
case 'application/gzip':
|
||||||
|
format = 'gz';
|
||||||
|
isCompressed = true;
|
||||||
|
isArchive = true;
|
||||||
|
break;
|
||||||
|
case 'application/zip':
|
||||||
|
format = 'zip';
|
||||||
|
isCompressed = true;
|
||||||
|
isArchive = true;
|
||||||
|
break;
|
||||||
|
case 'application/x-tar':
|
||||||
|
format = 'tar';
|
||||||
|
isArchive = true;
|
||||||
|
break;
|
||||||
|
case 'application/x-bzip2':
|
||||||
|
format = 'bz2';
|
||||||
|
isCompressed = true;
|
||||||
|
isArchive = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
format,
|
||||||
|
isCompressed,
|
||||||
|
isArchive,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all entries in the archive
|
||||||
|
*/
|
||||||
|
public async list(): Promise<IArchiveEntryInfo[]> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
const entries: IArchiveEntryInfo[] = [];
|
||||||
|
const streamFiles = await this.toStreamFiles();
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
streamFiles.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||||
|
entries.push({
|
||||||
|
path: streamFile.relativeFilePath || 'unknown',
|
||||||
|
size: 0, // Size not available without reading
|
||||||
|
isDirectory: false,
|
||||||
|
isFile: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
streamFiles.on('end', () => resolve(entries));
|
||||||
|
streamFiles.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if a specific file exists in the archive
|
||||||
|
*/
|
||||||
|
public async hasFile(filePath: string): Promise<boolean> {
|
||||||
|
this.ensureExtractionSource();
|
||||||
|
const entries = await this.list();
|
||||||
|
return entries.some((e) => e.path === filePath || e.path.endsWith(filePath));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ============================================
|
||||||
|
// PRIVATE HELPERS
|
||||||
|
// ============================================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure we're not in create mode when calling extraction methods
|
||||||
|
*/
|
||||||
|
private ensureNotInCreateMode(methodName: string): void {
|
||||||
|
if (this._mode === 'create') {
|
||||||
|
throw new Error(
|
||||||
|
`Cannot call .${methodName}() in creation mode. ` +
|
||||||
|
`Use extraction methods (.url(), .file(), .stream(), .buffer()) for extraction mode.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure we're not in extract mode when calling creation methods
|
||||||
|
*/
|
||||||
|
private ensureNotInExtractMode(methodName: string): void {
|
||||||
|
if (this._mode === 'extract') {
|
||||||
|
throw new Error(
|
||||||
|
`Cannot call .${methodName}() in extraction mode. ` +
|
||||||
|
`Use .format() for creation mode.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure an extraction source is configured
|
||||||
|
*/
|
||||||
|
private ensureExtractionSource(): void {
|
||||||
|
if (!this.sourceUrl && !this.sourceFilePath && !this.sourceStream && !this.archiveBuffer) {
|
||||||
|
throw new Error(
|
||||||
|
'No source configured. Call .url(), .file(), .stream(), or .buffer() first.'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensure a format is configured for creation
|
||||||
|
*/
|
||||||
|
private ensureCreationFormat(): void {
|
||||||
|
if (!this.creationFormat) {
|
||||||
|
throw new Error('No format specified. Call .format() before creating archive.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the source stream
|
||||||
|
*/
|
||||||
|
private async getSourceStream(): Promise<plugins.stream.Readable> {
|
||||||
|
if (this.archiveBuffer) {
|
||||||
|
return plugins.stream.Readable.from(this.archiveBuffer);
|
||||||
|
}
|
||||||
|
if (this.sourceStream) {
|
||||||
|
return this.sourceStream;
|
||||||
|
}
|
||||||
|
if (this.sourceUrl) {
|
||||||
|
const response = await plugins.smartrequest.SmartRequest.create()
|
||||||
|
.url(this.sourceUrl)
|
||||||
|
.get();
|
||||||
|
const webStream = response.stream();
|
||||||
|
return plugins.stream.Readable.fromWeb(webStream as any);
|
||||||
|
}
|
||||||
|
if (this.sourceFilePath) {
|
||||||
|
return plugins.fs.createReadStream(this.sourceFilePath);
|
||||||
|
}
|
||||||
|
throw new Error('No archive source configured');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a combined filter function from all configured filters
|
||||||
|
*/
|
||||||
|
private buildFilterFunction(): TEntryFilter | undefined {
|
||||||
|
const hasFilters =
|
||||||
|
this._filters.length > 0 ||
|
||||||
|
this._includePatterns.length > 0 ||
|
||||||
|
this._excludePatterns.length > 0;
|
||||||
|
|
||||||
|
if (!hasFilters) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (entry: IArchiveEntryInfo) => {
|
||||||
|
// Check include patterns (if any specified, at least one must match)
|
||||||
|
if (this._includePatterns.length > 0) {
|
||||||
|
const included = this._includePatterns.some((p) => p.test(entry.path));
|
||||||
|
if (!included) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check exclude patterns (none must match)
|
||||||
|
for (const pattern of this._excludePatterns) {
|
||||||
|
if (pattern.test(entry.path)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check custom filters (all must pass)
|
||||||
|
for (const filter of this._filters) {
|
||||||
|
if (!filter(entry)) return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve pending directories to entries
|
||||||
|
*/
|
||||||
|
private async resolveDirectories(): Promise<void> {
|
||||||
|
for (const dir of this.pendingDirectories) {
|
||||||
|
const files = await plugins.listFileTree(dir.sourcePath, '**/*');
|
||||||
|
for (const filePath of files) {
|
||||||
|
const archivePath = dir.archiveBase
|
||||||
|
? plugins.path.join(dir.archiveBase, filePath)
|
||||||
|
: filePath;
|
||||||
|
const absolutePath = plugins.path.join(dir.sourcePath, filePath);
|
||||||
|
const content = await plugins.fsPromises.readFile(absolutePath);
|
||||||
|
this.pendingEntries.push({
|
||||||
|
archivePath,
|
||||||
|
content,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.pendingDirectories = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get entries filtered by include/exclude patterns
|
||||||
|
*/
|
||||||
|
private getFilteredEntries(): IArchiveEntry[] {
|
||||||
|
const filterFn = this.buildFilterFunction();
|
||||||
|
if (!filterFn) {
|
||||||
|
return this.pendingEntries;
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.pendingEntries.filter((entry) =>
|
||||||
|
filterFn({
|
||||||
|
path: entry.archivePath,
|
||||||
|
size: 0,
|
||||||
|
isDirectory: false,
|
||||||
|
isFile: true,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a stream to buffer
|
||||||
|
*/
|
||||||
|
private async streamToBuffer(stream: plugins.stream.Readable): Promise<Buffer> {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
stream.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
||||||
|
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||||
|
stream.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read first chunk from stream
|
||||||
|
*/
|
||||||
|
private async readFirstChunk(stream: plugins.stream.Readable): Promise<Buffer> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const onData = (chunk: Buffer) => {
|
||||||
|
stream.removeListener('data', onData);
|
||||||
|
stream.removeListener('error', reject);
|
||||||
|
resolve(chunk);
|
||||||
|
};
|
||||||
|
stream.on('data', onData);
|
||||||
|
stream.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
import type { SmartArchive } from './classes.smartarchive.js';
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
import type { IArchiveEntry, TCompressionLevel } from './interfaces.js';
|
||||||
|
import { GzipTools } from './classes.gziptools.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TAR archive creation and extraction utilities
|
||||||
|
*/
|
||||||
export class TarTools {
|
export class TarTools {
|
||||||
// INSTANCE
|
/**
|
||||||
constructor() {}
|
* Add a file to a TAR pack stream
|
||||||
|
*/
|
||||||
// packing
|
|
||||||
public async addFileToPack(
|
public async addFileToPack(
|
||||||
pack: plugins.tarStream.Pack,
|
pack: plugins.tarStream.Pack,
|
||||||
optionsArg: {
|
optionsArg: {
|
||||||
@@ -13,7 +16,7 @@ export class TarTools {
|
|||||||
content?:
|
content?:
|
||||||
| string
|
| string
|
||||||
| Buffer
|
| Buffer
|
||||||
| plugins.smartstream.stream.Readable
|
| plugins.stream.Readable
|
||||||
| plugins.smartfile.SmartFile
|
| plugins.smartfile.SmartFile
|
||||||
| plugins.smartfile.StreamFile;
|
| plugins.smartfile.StreamFile;
|
||||||
byteLength?: number;
|
byteLength?: number;
|
||||||
@@ -26,17 +29,20 @@ export class TarTools {
|
|||||||
if (optionsArg.fileName) {
|
if (optionsArg.fileName) {
|
||||||
fileName = optionsArg.fileName;
|
fileName = optionsArg.fileName;
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
||||||
fileName = (optionsArg.content as plugins.smartfile.SmartFile).relative;
|
fileName = optionsArg.content.relative;
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
||||||
fileName = (optionsArg.content as plugins.smartfile.StreamFile).relativeFilePath;
|
fileName = optionsArg.content.relativeFilePath;
|
||||||
} else if (optionsArg.filePath) {
|
} else if (optionsArg.filePath) {
|
||||||
fileName = optionsArg.filePath;
|
fileName = optionsArg.filePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
if (!fileName) {
|
||||||
* contentByteLength is used to set the size of the entry in the tar file
|
reject(new Error('No filename specified for TAR entry'));
|
||||||
*/
|
return;
|
||||||
let contentByteLength: number;
|
}
|
||||||
|
|
||||||
|
// Determine content byte length
|
||||||
|
let contentByteLength: number | undefined;
|
||||||
if (optionsArg.byteLength) {
|
if (optionsArg.byteLength) {
|
||||||
contentByteLength = optionsArg.byteLength;
|
contentByteLength = optionsArg.byteLength;
|
||||||
} else if (typeof optionsArg.content === 'string') {
|
} else if (typeof optionsArg.content === 'string') {
|
||||||
@@ -44,44 +50,39 @@ export class TarTools {
|
|||||||
} else if (Buffer.isBuffer(optionsArg.content)) {
|
} else if (Buffer.isBuffer(optionsArg.content)) {
|
||||||
contentByteLength = optionsArg.content.length;
|
contentByteLength = optionsArg.content.length;
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
||||||
contentByteLength = await optionsArg.content.getSize(); // assuming SmartFile has getSize method
|
contentByteLength = await optionsArg.content.getSize();
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
||||||
contentByteLength = await optionsArg.content.getSize(); // assuming StreamFile has getSize method
|
contentByteLength = await optionsArg.content.getSize();
|
||||||
} else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) {
|
|
||||||
console.warn(
|
|
||||||
'@push.rocks/smartarchive: When streaming, it is recommended to provide byteLength, if known.'
|
|
||||||
);
|
|
||||||
} else if (optionsArg.filePath) {
|
} else if (optionsArg.filePath) {
|
||||||
const fileStat = await plugins.smartfile.fs.stat(optionsArg.filePath);
|
const fileStat = await plugins.fsPromises.stat(optionsArg.filePath);
|
||||||
contentByteLength = fileStat.size;
|
contentByteLength = fileStat.size;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// Convert all content types to Readable stream
|
||||||
* here we try to harmonize all kind of entries towards a readable stream
|
let content: plugins.stream.Readable;
|
||||||
*/
|
|
||||||
let content: plugins.smartstream.stream.Readable;
|
|
||||||
if (Buffer.isBuffer(optionsArg.content)) {
|
if (Buffer.isBuffer(optionsArg.content)) {
|
||||||
content = plugins.smartstream.stream.Readable.from(optionsArg.content);
|
content = plugins.stream.Readable.from(optionsArg.content);
|
||||||
} else if (typeof optionsArg.content === 'string') {
|
} else if (typeof optionsArg.content === 'string') {
|
||||||
content = plugins.smartstream.stream.Readable.from(Buffer.from(optionsArg.content));
|
content = plugins.stream.Readable.from(Buffer.from(optionsArg.content));
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.SmartFile) {
|
||||||
content = plugins.smartstream.stream.Readable.from(optionsArg.content.contents);
|
content = plugins.stream.Readable.from(optionsArg.content.contents);
|
||||||
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
} else if (optionsArg.content instanceof plugins.smartfile.StreamFile) {
|
||||||
content = await optionsArg.content.createReadStream();
|
content = await optionsArg.content.createReadStream();
|
||||||
} else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) {
|
} else if (optionsArg.content instanceof plugins.stream.Readable) {
|
||||||
content = optionsArg.content;
|
content = optionsArg.content;
|
||||||
|
} else if (optionsArg.filePath) {
|
||||||
|
content = plugins.fs.createReadStream(optionsArg.filePath);
|
||||||
|
} else {
|
||||||
|
reject(new Error('No content or filePath specified for TAR entry'));
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const entry = pack.entry(
|
const entry = pack.entry(
|
||||||
{
|
{
|
||||||
name: fileName,
|
name: fileName,
|
||||||
...(contentByteLength
|
...(contentByteLength !== undefined ? { size: contentByteLength } : {}),
|
||||||
? {
|
|
||||||
size: contentByteLength,
|
|
||||||
}
|
|
||||||
: null),
|
|
||||||
},
|
},
|
||||||
(err: Error) => {
|
(err: Error | null) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
reject(err);
|
reject(err);
|
||||||
} else {
|
} else {
|
||||||
@@ -91,37 +92,117 @@ export class TarTools {
|
|||||||
);
|
);
|
||||||
|
|
||||||
content.pipe(entry);
|
content.pipe(entry);
|
||||||
resolve();
|
// Note: resolve() is called in the callback above when pipe completes
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* packs a directory from disk into a tar stream
|
* Pack a directory into a TAR stream
|
||||||
* @param directoryPath
|
|
||||||
*/
|
*/
|
||||||
public async packDirectory(directoryPath: string) {
|
public async packDirectory(directoryPath: string): Promise<plugins.tarStream.Pack> {
|
||||||
const fileTree = await plugins.smartfile.fs.listFileTree(directoryPath, '**/*');
|
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
|
||||||
const pack = await this.getPackStream();
|
const pack = await this.getPackStream();
|
||||||
|
|
||||||
for (const filePath of fileTree) {
|
for (const filePath of fileTree) {
|
||||||
const absolutePath = plugins.path.join(directoryPath, filePath);
|
const absolutePath = plugins.path.join(directoryPath, filePath);
|
||||||
const fileStat = await plugins.smartfile.fs.stat(absolutePath);
|
const fileStat = await plugins.fsPromises.stat(absolutePath);
|
||||||
await this.addFileToPack(pack, {
|
await this.addFileToPack(pack, {
|
||||||
byteLength: fileStat.size,
|
byteLength: fileStat.size,
|
||||||
filePath: absolutePath,
|
filePath: absolutePath,
|
||||||
fileName: filePath,
|
fileName: filePath,
|
||||||
content: plugins.smartfile.fsStream.createReadStream(absolutePath),
|
content: plugins.fs.createReadStream(absolutePath),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return pack;
|
return pack;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getPackStream() {
|
/**
|
||||||
const pack = plugins.tarStream.pack();
|
* Get a new TAR pack stream
|
||||||
return pack;
|
*/
|
||||||
|
public async getPackStream(): Promise<plugins.tarStream.Pack> {
|
||||||
|
return plugins.tarStream.pack();
|
||||||
}
|
}
|
||||||
|
|
||||||
// extracting
|
/**
|
||||||
getDecompressionStream() {
|
* Get a TAR extraction stream
|
||||||
|
*/
|
||||||
|
public getDecompressionStream(): plugins.tarStream.Extract {
|
||||||
return plugins.tarStream.extract();
|
return plugins.tarStream.extract();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pack files into a TAR buffer
|
||||||
|
*/
|
||||||
|
public async packFiles(files: IArchiveEntry[]): Promise<Buffer> {
|
||||||
|
const pack = await this.getPackStream();
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
await this.addFileToPack(pack, {
|
||||||
|
fileName: file.archivePath,
|
||||||
|
content: file.content as string | Buffer | plugins.stream.Readable | plugins.smartfile.SmartFile | plugins.smartfile.StreamFile,
|
||||||
|
byteLength: file.size,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pack.finalize();
|
||||||
|
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
pack.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
pack.on('end', () => resolve(Buffer.concat(chunks)));
|
||||||
|
pack.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pack a directory into a TAR.GZ buffer
|
||||||
|
*/
|
||||||
|
public async packDirectoryToTarGz(
|
||||||
|
directoryPath: string,
|
||||||
|
compressionLevel?: TCompressionLevel
|
||||||
|
): Promise<Buffer> {
|
||||||
|
const pack = await this.packDirectory(directoryPath);
|
||||||
|
pack.finalize();
|
||||||
|
|
||||||
|
const gzipTools = new GzipTools();
|
||||||
|
const gzipStream = gzipTools.getCompressionStream(compressionLevel);
|
||||||
|
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
pack
|
||||||
|
.pipe(gzipStream)
|
||||||
|
.on('data', (chunk: Buffer) => chunks.push(chunk))
|
||||||
|
.on('end', () => resolve(Buffer.concat(chunks)))
|
||||||
|
.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pack a directory into a TAR.GZ stream
|
||||||
|
*/
|
||||||
|
public async packDirectoryToTarGzStream(
|
||||||
|
directoryPath: string,
|
||||||
|
compressionLevel?: TCompressionLevel
|
||||||
|
): Promise<plugins.stream.Readable> {
|
||||||
|
const pack = await this.packDirectory(directoryPath);
|
||||||
|
pack.finalize();
|
||||||
|
|
||||||
|
const gzipTools = new GzipTools();
|
||||||
|
const gzipStream = gzipTools.getCompressionStream(compressionLevel);
|
||||||
|
|
||||||
|
return pack.pipe(gzipStream);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pack files into a TAR.GZ buffer
|
||||||
|
*/
|
||||||
|
public async packFilesToTarGz(
|
||||||
|
files: IArchiveEntry[],
|
||||||
|
compressionLevel?: TCompressionLevel
|
||||||
|
): Promise<Buffer> {
|
||||||
|
const tarBuffer = await this.packFiles(files);
|
||||||
|
const gzipTools = new GzipTools();
|
||||||
|
return gzipTools.compress(tarBuffer, compressionLevel);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,74 +1,196 @@
|
|||||||
import type { SmartArchive } from './classes.smartarchive.js';
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
import type { IArchiveEntry, TCompressionLevel } from './interfaces.js';
|
||||||
|
|
||||||
class DecompressZipTransform extends plugins.smartstream.SmartDuplex<ArrayBufferLike> {
|
/**
|
||||||
private streamtools: plugins.smartstream.IStreamTools;
|
* Transform stream for ZIP decompression using fflate
|
||||||
|
* Emits StreamFile objects for each file in the archive
|
||||||
|
*/
|
||||||
|
export class ZipDecompressionTransform extends plugins.smartstream.SmartDuplex<Buffer, plugins.smartfile.StreamFile> {
|
||||||
|
private streamtools!: plugins.smartstream.IStreamTools;
|
||||||
private unzipper = new plugins.fflate.Unzip(async (fileArg) => {
|
private unzipper = new plugins.fflate.Unzip(async (fileArg) => {
|
||||||
let resultBuffer: Buffer;
|
let resultBuffer: Buffer;
|
||||||
fileArg.ondata = async (flateError, dat, final) => {
|
fileArg.ondata = async (_flateError, dat, final) => {
|
||||||
resultBuffer? resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)])
|
resultBuffer
|
||||||
: resultBuffer = Buffer.from(dat);
|
? (resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)]))
|
||||||
|
: (resultBuffer = Buffer.from(dat));
|
||||||
if (final) {
|
if (final) {
|
||||||
const streamFile = plugins.smartfile.StreamFile.fromBuffer(resultBuffer);
|
const streamFile = plugins.smartfile.StreamFile.fromBuffer(resultBuffer);
|
||||||
streamFile.relativeFilePath = fileArg.name;
|
streamFile.relativeFilePath = fileArg.name;
|
||||||
this.streamtools.push(streamFile);
|
this.streamtools.push(streamFile);
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
fileArg.start();
|
fileArg.start();
|
||||||
});
|
});
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super({
|
super({
|
||||||
objectMode: true,
|
objectMode: true,
|
||||||
writeFunction: async (chunkArg: Buffer, streamtoolsArg) => {
|
writeFunction: async (chunkArg, streamtoolsArg) => {
|
||||||
this.streamtools? null : this.streamtools = streamtoolsArg;
|
this.streamtools ? null : (this.streamtools = streamtoolsArg);
|
||||||
this.unzipper.push(chunkArg, false);
|
this.unzipper.push(
|
||||||
|
Buffer.isBuffer(chunkArg) ? chunkArg : Buffer.from(chunkArg as unknown as ArrayBuffer),
|
||||||
|
false
|
||||||
|
);
|
||||||
|
return null;
|
||||||
},
|
},
|
||||||
finalFunction: async () => {
|
finalFunction: async () => {
|
||||||
this.unzipper.push(Buffer.from(''), true);
|
this.unzipper.push(Buffer.from(''), true);
|
||||||
await plugins.smartdelay.delayFor(0);
|
await plugins.smartdelay.delayFor(0);
|
||||||
await this.streamtools.push(null);
|
await this.streamtools.push(null);
|
||||||
}
|
return null;
|
||||||
|
},
|
||||||
});
|
});
|
||||||
this.unzipper.register(plugins.fflate.UnzipInflate);
|
this.unzipper.register(plugins.fflate.UnzipInflate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// This class wraps fflate's zip in a Node.js Transform stream for compression
|
/**
|
||||||
export class CompressZipTransform extends plugins.stream.Transform {
|
* Streaming ZIP compression using fflate
|
||||||
files: { [fileName: string]: Uint8Array };
|
* Allows adding multiple entries before finalizing
|
||||||
|
*/
|
||||||
|
export class ZipCompressionStream extends plugins.stream.Duplex {
|
||||||
|
private files: Map<string, { data: Uint8Array; options?: plugins.fflate.ZipOptions }> = new Map();
|
||||||
|
private finalized = false;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
this.files = {};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) {
|
/**
|
||||||
// Simple example: storing chunks in memory before finalizing ZIP in _flush
|
* Add a file entry to the ZIP archive
|
||||||
this.files['file.txt'] = new Uint8Array(chunk);
|
*/
|
||||||
callback();
|
public async addEntry(
|
||||||
}
|
fileName: string,
|
||||||
|
content: Buffer | plugins.stream.Readable,
|
||||||
|
options?: { compressionLevel?: TCompressionLevel }
|
||||||
|
): Promise<void> {
|
||||||
|
if (this.finalized) {
|
||||||
|
throw new Error('Cannot add entries to a finalized ZIP archive');
|
||||||
|
}
|
||||||
|
|
||||||
_flush(callback: plugins.stream.TransformCallback) {
|
let data: Buffer;
|
||||||
plugins.fflate.zip(this.files, (err, zipped) => {
|
if (Buffer.isBuffer(content)) {
|
||||||
if (err) {
|
data = content;
|
||||||
callback(err);
|
} else {
|
||||||
} else {
|
// Collect stream to buffer
|
||||||
this.push(Buffer.from(zipped));
|
const chunks: Buffer[] = [];
|
||||||
callback();
|
for await (const chunk of content) {
|
||||||
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||||
}
|
}
|
||||||
|
data = Buffer.concat(chunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.files.set(fileName, {
|
||||||
|
data: new Uint8Array(data),
|
||||||
|
options: options?.compressionLevel !== undefined ? { level: options.compressionLevel } : undefined,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Finalize the ZIP archive and emit the compressed data
|
||||||
|
*/
|
||||||
|
public async finalize(): Promise<void> {
|
||||||
|
if (this.finalized) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.finalized = true;
|
||||||
|
|
||||||
|
const filesObj: plugins.fflate.Zippable = {};
|
||||||
|
for (const [name, { data, options }] of this.files) {
|
||||||
|
filesObj[name] = options ? [data, options] : data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||||
|
try {
|
||||||
|
const result = plugins.fflate.zipSync(filesObj);
|
||||||
|
this.push(Buffer.from(result));
|
||||||
|
this.push(null);
|
||||||
|
} catch (err) {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_read(): void {
|
||||||
|
// No-op: data is pushed when finalize() is called
|
||||||
|
}
|
||||||
|
|
||||||
|
_write(
|
||||||
|
_chunk: Buffer,
|
||||||
|
_encoding: BufferEncoding,
|
||||||
|
callback: (error?: Error | null) => void
|
||||||
|
): void {
|
||||||
|
// Not used for ZIP creation - use addEntry() instead
|
||||||
|
callback(new Error('Use addEntry() to add files to the ZIP archive'));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ZIP compression and decompression utilities
|
||||||
|
*/
|
||||||
export class ZipTools {
|
export class ZipTools {
|
||||||
constructor() {
|
/**
|
||||||
|
* Get a streaming compression object for creating ZIP archives
|
||||||
|
*/
|
||||||
|
public getCompressionStream(): ZipCompressionStream {
|
||||||
|
return new ZipCompressionStream();
|
||||||
}
|
}
|
||||||
|
|
||||||
public getCompressionStream() {
|
/**
|
||||||
return new CompressZipTransform();
|
* Get a streaming decompression transform for extracting ZIP archives
|
||||||
|
*/
|
||||||
|
public getDecompressionStream(): ZipDecompressionTransform {
|
||||||
|
return new ZipDecompressionTransform();
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDecompressionStream() {
|
/**
|
||||||
return new DecompressZipTransform();
|
* Create a ZIP archive from an array of entries
|
||||||
|
*/
|
||||||
|
public async createZip(entries: IArchiveEntry[], compressionLevel?: TCompressionLevel): Promise<Buffer> {
|
||||||
|
const filesObj: plugins.fflate.Zippable = {};
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
let data: Uint8Array;
|
||||||
|
|
||||||
|
if (typeof entry.content === 'string') {
|
||||||
|
data = new TextEncoder().encode(entry.content);
|
||||||
|
} else if (Buffer.isBuffer(entry.content)) {
|
||||||
|
data = new Uint8Array(entry.content);
|
||||||
|
} else if (entry.content instanceof plugins.smartfile.SmartFile) {
|
||||||
|
data = new Uint8Array(entry.content.contents);
|
||||||
|
} else if (entry.content instanceof plugins.smartfile.StreamFile) {
|
||||||
|
const buffer = await entry.content.getContentAsBuffer();
|
||||||
|
data = new Uint8Array(buffer);
|
||||||
|
} else {
|
||||||
|
// Readable stream
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
for await (const chunk of entry.content as plugins.stream.Readable) {
|
||||||
|
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||||
|
}
|
||||||
|
data = new Uint8Array(Buffer.concat(chunks));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (compressionLevel !== undefined) {
|
||||||
|
filesObj[entry.archivePath] = [data, { level: compressionLevel }];
|
||||||
|
} else {
|
||||||
|
filesObj[entry.archivePath] = data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||||
|
const result = plugins.fflate.zipSync(filesObj);
|
||||||
|
return Buffer.from(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract a ZIP buffer to an array of entries
|
||||||
|
*/
|
||||||
|
public async extractZip(data: Buffer): Promise<Array<{ path: string; content: Buffer }>> {
|
||||||
|
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||||
|
const result = plugins.fflate.unzipSync(data);
|
||||||
|
const entries: Array<{ path: string; content: Buffer }> = [];
|
||||||
|
for (const [path, content] of Object.entries(result)) {
|
||||||
|
entries.push({ path, content: Buffer.from(content) });
|
||||||
|
}
|
||||||
|
return entries;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
70
ts/errors.ts
Normal file
70
ts/errors.ts
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
/**
|
||||||
|
* Base error class for smartarchive
|
||||||
|
*/
|
||||||
|
export class SmartArchiveError extends Error {
|
||||||
|
public readonly code: string;
|
||||||
|
|
||||||
|
constructor(message: string, code: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = 'SmartArchiveError';
|
||||||
|
this.code = code;
|
||||||
|
// Maintains proper stack trace for where error was thrown (V8)
|
||||||
|
if (Error.captureStackTrace) {
|
||||||
|
Error.captureStackTrace(this, this.constructor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* BZIP2-specific decompression errors
|
||||||
|
*/
|
||||||
|
export class Bzip2Error extends SmartArchiveError {
|
||||||
|
constructor(message: string, code: string = 'BZIP2_ERROR') {
|
||||||
|
super(message, code);
|
||||||
|
this.name = 'Bzip2Error';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Archive format detection errors
|
||||||
|
*/
|
||||||
|
export class ArchiveFormatError extends SmartArchiveError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message, 'ARCHIVE_FORMAT_ERROR');
|
||||||
|
this.name = 'ArchiveFormatError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stream processing errors
|
||||||
|
*/
|
||||||
|
export class StreamError extends SmartArchiveError {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message, 'STREAM_ERROR');
|
||||||
|
this.name = 'StreamError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* BZIP2 error codes for programmatic error handling
|
||||||
|
*/
|
||||||
|
export const BZIP2_ERROR_CODES = {
|
||||||
|
NO_MAGIC_NUMBER: 'BZIP2_NO_MAGIC',
|
||||||
|
INVALID_ARCHIVE: 'BZIP2_INVALID_ARCHIVE',
|
||||||
|
CRC_MISMATCH: 'BZIP2_CRC_MISMATCH',
|
||||||
|
INVALID_BLOCK_DATA: 'BZIP2_INVALID_BLOCK',
|
||||||
|
BUFFER_OVERFLOW: 'BZIP2_BUFFER_OVERFLOW',
|
||||||
|
INVALID_HUFFMAN: 'BZIP2_INVALID_HUFFMAN',
|
||||||
|
INVALID_SELECTOR: 'BZIP2_INVALID_SELECTOR',
|
||||||
|
INVALID_POSITION: 'BZIP2_INVALID_POSITION',
|
||||||
|
PREMATURE_END: 'BZIP2_PREMATURE_END',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type TBzip2ErrorCode = typeof BZIP2_ERROR_CODES[keyof typeof BZIP2_ERROR_CODES];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Throw a BZIP2 error with a specific code
|
||||||
|
*/
|
||||||
|
export function throwBzip2Error(message: string, code: TBzip2ErrorCode): never {
|
||||||
|
throw new Bzip2Error(message, code);
|
||||||
|
}
|
||||||
11
ts/index.ts
11
ts/index.ts
@@ -1,4 +1,15 @@
|
|||||||
|
// Core types and errors
|
||||||
|
export * from './interfaces.js';
|
||||||
|
export * from './errors.js';
|
||||||
|
|
||||||
|
// Main archive class
|
||||||
export * from './classes.smartarchive.js';
|
export * from './classes.smartarchive.js';
|
||||||
|
|
||||||
|
// Format-specific tools
|
||||||
export * from './classes.tartools.js';
|
export * from './classes.tartools.js';
|
||||||
export * from './classes.ziptools.js';
|
export * from './classes.ziptools.js';
|
||||||
export * from './classes.gziptools.js';
|
export * from './classes.gziptools.js';
|
||||||
|
export * from './classes.bzip2tools.js';
|
||||||
|
|
||||||
|
// Archive analysis
|
||||||
|
export * from './classes.archiveanalyzer.js';
|
||||||
|
|||||||
136
ts/interfaces.ts
Normal file
136
ts/interfaces.ts
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
import type * as stream from 'node:stream';
|
||||||
|
import type { SmartFile, StreamFile } from '@push.rocks/smartfile';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Supported archive formats
|
||||||
|
*/
|
||||||
|
export type TArchiveFormat = 'tar' | 'tar.gz' | 'tgz' | 'zip' | 'gz' | 'bz2';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compression level (0 = no compression, 9 = maximum compression)
|
||||||
|
*/
|
||||||
|
export type TCompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Supported MIME types for archive detection
|
||||||
|
*/
|
||||||
|
export type TSupportedMime =
|
||||||
|
| 'application/gzip'
|
||||||
|
| 'application/zip'
|
||||||
|
| 'application/x-bzip2'
|
||||||
|
| 'application/x-tar'
|
||||||
|
| undefined;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entry to add to an archive during creation
|
||||||
|
*/
|
||||||
|
export interface IArchiveEntry {
|
||||||
|
/** Path within the archive */
|
||||||
|
archivePath: string;
|
||||||
|
/** Content: string, Buffer, Readable stream, SmartFile, or StreamFile */
|
||||||
|
content: string | Buffer | stream.Readable | SmartFile | StreamFile;
|
||||||
|
/** Optional size hint for streams (improves performance) */
|
||||||
|
size?: number;
|
||||||
|
/** Optional file mode/permissions */
|
||||||
|
mode?: number;
|
||||||
|
/** Optional modification time */
|
||||||
|
mtime?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options for creating archives
|
||||||
|
*/
|
||||||
|
export interface IArchiveCreationOptions {
|
||||||
|
/** Target archive format */
|
||||||
|
format: TArchiveFormat;
|
||||||
|
/** Compression level (0-9, default depends on format) */
|
||||||
|
compressionLevel?: TCompressionLevel;
|
||||||
|
/** Base path to strip from file paths in archive */
|
||||||
|
basePath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options for extracting archives
|
||||||
|
*/
|
||||||
|
export interface IArchiveExtractionOptions {
|
||||||
|
/** Target directory for extraction */
|
||||||
|
targetDir: string;
|
||||||
|
/** Optional filename for single-file archives (gz, bz2) */
|
||||||
|
fileName?: string;
|
||||||
|
/** Number of leading path components to strip */
|
||||||
|
stripComponents?: number;
|
||||||
|
/** Filter function to select which entries to extract */
|
||||||
|
filter?: (entry: IArchiveEntryInfo) => boolean;
|
||||||
|
/** Whether to overwrite existing files */
|
||||||
|
overwrite?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Information about an archive entry
|
||||||
|
*/
|
||||||
|
export interface IArchiveEntryInfo {
|
||||||
|
/** Path of the entry within the archive */
|
||||||
|
path: string;
|
||||||
|
/** Size in bytes */
|
||||||
|
size: number;
|
||||||
|
/** Whether this entry is a directory */
|
||||||
|
isDirectory: boolean;
|
||||||
|
/** Whether this entry is a file */
|
||||||
|
isFile: boolean;
|
||||||
|
/** Modification time */
|
||||||
|
mtime?: Date;
|
||||||
|
/** File mode/permissions */
|
||||||
|
mode?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Result of archive analysis
|
||||||
|
*/
|
||||||
|
export interface IArchiveInfo {
|
||||||
|
/** Detected archive format */
|
||||||
|
format: TArchiveFormat | null;
|
||||||
|
/** Whether the archive is compressed */
|
||||||
|
isCompressed: boolean;
|
||||||
|
/** Whether this is a recognized archive format */
|
||||||
|
isArchive: boolean;
|
||||||
|
/** List of entries (if available without full extraction) */
|
||||||
|
entries?: IArchiveEntryInfo[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Options for adding a file to a TAR pack stream
|
||||||
|
*/
|
||||||
|
export interface IAddFileOptions {
|
||||||
|
/** Filename within the archive */
|
||||||
|
fileName?: string;
|
||||||
|
/** File content */
|
||||||
|
content?: string | Buffer | stream.Readable | SmartFile | StreamFile;
|
||||||
|
/** Size in bytes (required for streams) */
|
||||||
|
byteLength?: number;
|
||||||
|
/** Path to file on disk (alternative to content) */
|
||||||
|
filePath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bit reader interface for BZIP2 decompression
|
||||||
|
*/
|
||||||
|
export interface IBitReader {
|
||||||
|
(n: number | null): number | void;
|
||||||
|
bytesRead: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Huffman group for BZIP2 decompression
|
||||||
|
*/
|
||||||
|
export interface IHuffmanGroup {
|
||||||
|
permute: Int32Array;
|
||||||
|
limit: Int32Array;
|
||||||
|
base: Int32Array;
|
||||||
|
minLen: number;
|
||||||
|
maxLen: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entry filter predicate for fluent API
|
||||||
|
*/
|
||||||
|
export type TEntryFilter = (entry: IArchiveEntryInfo) => boolean;
|
||||||
@@ -2,6 +2,6 @@ import * as plugins from './plugins.js';
|
|||||||
|
|
||||||
export const packageDir = plugins.path.join(
|
export const packageDir = plugins.path.join(
|
||||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||||
'../'
|
'../',
|
||||||
);
|
);
|
||||||
export const nogitDir = plugins.path.join(packageDir, './.nogit');
|
export const nogitDir = plugins.path.join(packageDir, './.nogit');
|
||||||
|
|||||||
@@ -1,8 +1,34 @@
|
|||||||
// node native scope
|
// node native scope
|
||||||
import * as path from 'path';
|
import * as path from 'node:path';
|
||||||
import * as stream from 'stream';
|
import * as stream from 'node:stream';
|
||||||
|
import * as fs from 'node:fs';
|
||||||
|
import * as fsPromises from 'node:fs/promises';
|
||||||
|
|
||||||
export { path, stream };
|
export { path, stream, fs, fsPromises };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List files in a directory recursively, returning relative paths
|
||||||
|
*/
|
||||||
|
export async function listFileTree(dirPath: string, _pattern: string = '**/*'): Promise<string[]> {
|
||||||
|
const results: string[] = [];
|
||||||
|
|
||||||
|
async function walkDir(currentPath: string, relativePath: string = '') {
|
||||||
|
const entries = await fsPromises.readdir(currentPath, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
const entryRelPath = relativePath ? path.join(relativePath, entry.name) : entry.name;
|
||||||
|
const entryFullPath = path.join(currentPath, entry.name);
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
await walkDir(entryFullPath, entryRelPath);
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
results.push(entryRelPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await walkDir(dirPath);
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
// @pushrocks scope
|
// @pushrocks scope
|
||||||
import * as smartfile from '@push.rocks/smartfile';
|
import * as smartfile from '@push.rocks/smartfile';
|
||||||
@@ -15,7 +41,17 @@ import * as smartstream from '@push.rocks/smartstream';
|
|||||||
import * as smartrx from '@push.rocks/smartrx';
|
import * as smartrx from '@push.rocks/smartrx';
|
||||||
import * as smarturl from '@push.rocks/smarturl';
|
import * as smarturl from '@push.rocks/smarturl';
|
||||||
|
|
||||||
export { smartfile, smartdelay, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl };
|
export {
|
||||||
|
smartfile,
|
||||||
|
smartdelay,
|
||||||
|
smartpath,
|
||||||
|
smartpromise,
|
||||||
|
smartrequest,
|
||||||
|
smartunique,
|
||||||
|
smartstream,
|
||||||
|
smartrx,
|
||||||
|
smarturl,
|
||||||
|
};
|
||||||
|
|
||||||
// third party scope
|
// third party scope
|
||||||
import * as fileType from 'file-type';
|
import * as fileType from 'file-type';
|
||||||
|
|||||||
@@ -6,9 +6,9 @@
|
|||||||
"module": "NodeNext",
|
"module": "NodeNext",
|
||||||
"moduleResolution": "NodeNext",
|
"moduleResolution": "NodeNext",
|
||||||
"esModuleInterop": true,
|
"esModuleInterop": true,
|
||||||
"verbatimModuleSyntax": true
|
"verbatimModuleSyntax": true,
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": {}
|
||||||
},
|
},
|
||||||
"exclude": [
|
"exclude": ["dist_*/**/*.d.ts"]
|
||||||
"dist_*/**/*.d.ts"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user