fix(ziptools,gziptools): Use fflate synchronous APIs for ZIP and GZIP operations for Deno compatibility; add TEntryFilter type and small docs/tests cleanup
This commit is contained in:
Binary file not shown.
@@ -1,68 +0,0 @@
|
||||
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
|
||||
# * For C, use cpp
|
||||
# * For JavaScript, use typescript
|
||||
# Special requirements:
|
||||
# * csharp: Requires the presence of a .sln file in the project folder.
|
||||
language: typescript
|
||||
|
||||
# whether to use the project's gitignore file to ignore files
|
||||
# Added on 2025-04-07
|
||||
ignore_all_files_in_gitignore: true
|
||||
# list of additional paths to ignore
|
||||
# same syntax as gitignore, so you can use * and **
|
||||
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||
# Added (renamed) on 2025-04-07
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
|
||||
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||
# Below is the complete list of tools for convenience.
|
||||
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||
# execute `uv run scripts/print_tool_overview.py`.
|
||||
#
|
||||
# * `activate_project`: Activates a project by name.
|
||||
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||
# * `delete_lines`: Deletes a range of lines within a file.
|
||||
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||
# * `execute_shell_command`: Executes a shell command.
|
||||
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||
# Should only be used in settings where the system prompt cannot be set,
|
||||
# e.g. in clients you have no control over, like Claude Desktop.
|
||||
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||
# * `read_file`: Reads a file within the project directory.
|
||||
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||
# * `remove_project`: Removes a project from the Serena configuration.
|
||||
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||
# * `switch_modes`: Activates modes by providing a list of their names
|
||||
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||
excluded_tools: []
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
project_name: "smartarchive"
|
||||
10
changelog.md
10
changelog.md
@@ -1,5 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-11-25 - 5.0.1 - fix(ziptools,gziptools)
|
||||
Use fflate synchronous APIs for ZIP and GZIP operations for Deno compatibility; add TEntryFilter type and small docs/tests cleanup
|
||||
|
||||
- Replace fflate async APIs (zip, unzip, gzip, gunzip with callbacks) with synchronous counterparts (zipSync, unzipSync, gzipSync, gunzipSync) to avoid Web Worker issues in Deno
|
||||
- ZipCompressionStream.finalize now uses fflate.zipSync and emits compressed Buffer synchronously
|
||||
- GzipTools.compress / decompress now delegate to compressSync / decompressSync for cross-runtime compatibility
|
||||
- ZipTools.createZip and ZipTools.extractZip now use zipSync/unzipSync and return Buffers
|
||||
- Add TEntryFilter type to ts/interfaces.ts for fluent API entry filtering
|
||||
- Minor readme.hints.md updates and small whitespace tidy in tests
|
||||
|
||||
## 2025-11-25 - 5.0.0 - BREAKING CHANGE(SmartArchive)
|
||||
Refactor public API: rename factory/extraction methods, introduce typed interfaces and improved compression tools
|
||||
|
||||
|
||||
@@ -1,38 +1,84 @@
|
||||
# Smartarchive Development Hints
|
||||
|
||||
## Dependency Upgrades (2025-01-25)
|
||||
## Architecture Overview
|
||||
|
||||
### Completed Upgrades
|
||||
- **@git.zone/tsbuild**: ^2.6.6 → ^3.1.0
|
||||
- **@git.zone/tsrun**: ^1.3.3 → ^2.0.0
|
||||
- **@git.zone/tstest**: ^2.3.4 → ^3.1.3
|
||||
- **@push.rocks/smartfile**: ^11.2.7 → ^13.0.0
|
||||
`@push.rocks/smartarchive` uses a **fluent builder pattern** for all archive operations. The main entry point is `SmartArchive.create()` which returns a builder instance.
|
||||
|
||||
### Migration Notes
|
||||
### Two Operating Modes
|
||||
|
||||
#### Smartfile v13 Migration
|
||||
Smartfile v13 removed filesystem operations (`fs`, `memory`, `fsStream` namespaces). These were replaced with Node.js native `fs` and `fs/promises`:
|
||||
1. **Extraction Mode** - Triggered by `.url()`, `.file()`, `.stream()`, or `.buffer()`
|
||||
2. **Creation Mode** - Triggered by `.format()` or `.entry()`
|
||||
|
||||
**Replacements made:**
|
||||
Modes are mutually exclusive - you cannot mix extraction and creation methods in the same chain.
|
||||
|
||||
## Key Classes
|
||||
|
||||
- **SmartArchive** - Main class with fluent API for all operations
|
||||
- **TarTools** - TAR-specific operations (pack/extract)
|
||||
- **ZipTools** - ZIP-specific operations using fflate
|
||||
- **GzipTools** - GZIP compression/decompression using fflate
|
||||
- **Bzip2Tools** - BZIP2 decompression (extract only, no creation)
|
||||
- **ArchiveAnalyzer** - Format detection via magic bytes
|
||||
|
||||
## Dependencies
|
||||
|
||||
- **fflate** - Pure JS compression for ZIP/GZIP (works in browser)
|
||||
- **tar-stream** - TAR archive handling
|
||||
- **file-type** - MIME type detection via magic bytes
|
||||
- **@push.rocks/smartfile** - SmartFile and StreamFile classes
|
||||
|
||||
## API Changes (v5.0.0)
|
||||
|
||||
The v5.0.0 release introduced a complete API refactor:
|
||||
|
||||
### Old API (deprecated)
|
||||
```typescript
|
||||
// Old static factory methods - NO LONGER EXIST
|
||||
await SmartArchive.fromUrl(url);
|
||||
await SmartArchive.fromFile(path);
|
||||
await SmartArchive.fromDirectory(path, options);
|
||||
```
|
||||
|
||||
### New Fluent API
|
||||
```typescript
|
||||
// Current fluent builder pattern
|
||||
await SmartArchive.create()
|
||||
.url(url)
|
||||
.extract(targetDir);
|
||||
|
||||
await SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.directory(path)
|
||||
.toFile(outputPath);
|
||||
```
|
||||
|
||||
## Migration Notes (from v4.x)
|
||||
|
||||
### Smartfile v13 Changes
|
||||
Smartfile v13 removed filesystem operations. Replacements:
|
||||
- `smartfile.fs.ensureDir(path)` → `fsPromises.mkdir(path, { recursive: true })`
|
||||
- `smartfile.fs.stat(path)` → `fsPromises.stat(path)`
|
||||
- `smartfile.fs.toReadStream(path)` → `fs.createReadStream(path)`
|
||||
- `smartfile.fs.toStringSync(path)` → `fsPromises.readFile(path, 'utf8')`
|
||||
- `smartfile.fs.listFileTree(dir, pattern)` → custom `listFileTree()` helper
|
||||
- `smartfile.fsStream.createReadStream(path)` → `fs.createReadStream(path)`
|
||||
- `smartfile.fsStream.createWriteStream(path)` → `fs.createWriteStream(path)`
|
||||
- `smartfile.memory.toFs(content, path)` → `fsPromises.writeFile(path, content)`
|
||||
|
||||
**Still using from smartfile v13:**
|
||||
### Still using from smartfile
|
||||
- `SmartFile` class (in-memory file representation)
|
||||
- `StreamFile` class (streaming file handling)
|
||||
|
||||
### Removed Dependencies
|
||||
- `through@2.3.8` - was unused in the codebase
|
||||
## Testing
|
||||
|
||||
## Architecture Notes
|
||||
```bash
|
||||
pnpm test # Run all tests
|
||||
tstest test/test.node+deno.ts --verbose # Run specific test
|
||||
```
|
||||
|
||||
- Uses `fflate` for ZIP/GZIP compression (pure JS, works in browser)
|
||||
- Uses `tar-stream` for TAR archive handling
|
||||
- Uses `file-type` for MIME type detection
|
||||
- Custom BZIP2 implementation in `ts/bzip2/` directory
|
||||
Tests use a Verdaccio registry URL (`verdaccio.lossless.digital`) for test archives.
|
||||
|
||||
## Key Files
|
||||
|
||||
- `ts/classes.smartarchive.ts` - Main SmartArchive class with fluent API
|
||||
- `ts/classes.tartools.ts` - TAR operations
|
||||
- `ts/classes.ziptools.ts` - ZIP operations
|
||||
- `ts/classes.gziptools.ts` - GZIP operations
|
||||
- `ts/classes.bzip2tools.ts` - BZIP2 decompression
|
||||
- `ts/classes.archiveanalyzer.ts` - Format detection
|
||||
- `ts/interfaces.ts` - Type definitions
|
||||
|
||||
712
readme.md
712
readme.md
@@ -1,8 +1,6 @@
|
||||
# @push.rocks/smartarchive 📦
|
||||
|
||||
Powerful archive manipulation for modern Node.js applications.
|
||||
|
||||
`@push.rocks/smartarchive` is a versatile library for handling archive files with a focus on developer experience. Work with **zip**, **tar**, **gzip**, and **bzip2** formats through a unified, streaming-optimized API.
|
||||
A powerful, streaming-first archive manipulation library with a fluent builder API. Works seamlessly in Node.js and Deno.
|
||||
|
||||
## Issue Reporting and Security
|
||||
|
||||
@@ -10,13 +8,15 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
|
||||
|
||||
## Features 🚀
|
||||
|
||||
- 📁 **Multi-format support** – Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, and `.bz2` archives
|
||||
- 📁 **Multi-format support** – Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, `.gz`, and `.bz2` archives
|
||||
- 🌊 **Streaming-first architecture** – Process large archives without memory constraints
|
||||
- 🔄 **Unified API** – Consistent interface across different archive formats
|
||||
- ✨ **Fluent builder API** – Chain methods for readable, expressive code
|
||||
- 🎯 **Smart detection** – Automatically identifies archive types via magic bytes
|
||||
- ⚡ **High performance** – Built on `tar-stream` and `fflate` for speed
|
||||
- 🔧 **Flexible I/O** – Work with files, URLs, and streams seamlessly
|
||||
- 🔧 **Flexible I/O** – Work with files, URLs, streams, and buffers seamlessly
|
||||
- 🛠️ **Modern TypeScript** – Full type safety and excellent IDE support
|
||||
- 🔄 **Dual-mode operation** – Extract existing archives OR create new ones
|
||||
- 🦕 **Cross-runtime** – Works in both Node.js and Deno environments
|
||||
|
||||
## Installation 📥
|
||||
|
||||
@@ -39,88 +39,288 @@ yarn add @push.rocks/smartarchive
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Extract a .tar.gz archive from a URL directly to the filesystem
|
||||
const archive = await SmartArchive.fromArchiveUrl(
|
||||
'https://registry.npmjs.org/some-package/-/some-package-1.0.0.tgz'
|
||||
);
|
||||
await archive.exportToFs('./extracted');
|
||||
await SmartArchive.create()
|
||||
.url('https://registry.npmjs.org/some-package/-/some-package-1.0.0.tgz')
|
||||
.extract('./extracted');
|
||||
```
|
||||
|
||||
### Process archive as a stream
|
||||
### Create an archive from entries
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Stream-based processing for memory efficiency
|
||||
const archive = await SmartArchive.fromArchiveFile('./large-archive.zip');
|
||||
const streamOfFiles = await archive.exportToStreamOfStreamFiles();
|
||||
// Create a tar.gz archive with files
|
||||
await SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.compression(6)
|
||||
.entry('config.json', JSON.stringify({ name: 'myapp' }))
|
||||
.entry('readme.txt', 'Hello World!')
|
||||
.toFile('./backup.tar.gz');
|
||||
```
|
||||
|
||||
// Process each file in the archive
|
||||
streamOfFiles.on('data', async (streamFile) => {
|
||||
console.log(`Processing ${streamFile.relativeFilePath}`);
|
||||
const readStream = await streamFile.createReadStream();
|
||||
// Handle individual file stream
|
||||
});
|
||||
### Extract with filtering and path manipulation
|
||||
|
||||
streamOfFiles.on('end', () => {
|
||||
console.log('Extraction complete');
|
||||
});
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Extract only JSON files, stripping the first path component
|
||||
await SmartArchive.create()
|
||||
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||
.stripComponents(1) // Remove 'package/' prefix
|
||||
.include(/\.json$/) // Only extract JSON files
|
||||
.extract('./node_modules/lodash');
|
||||
```
|
||||
|
||||
## Core Concepts 💡
|
||||
|
||||
### Archive Sources
|
||||
### Fluent Builder Pattern
|
||||
|
||||
`SmartArchive` accepts archives from three sources:
|
||||
`SmartArchive` uses a fluent builder pattern where you chain methods to configure the operation:
|
||||
|
||||
| Source | Method | Use Case |
|
||||
|--------|--------|----------|
|
||||
| **URL** | `SmartArchive.fromArchiveUrl(url)` | Download and process archives from the web |
|
||||
| **File** | `SmartArchive.fromArchiveFile(path)` | Load archives from the local filesystem |
|
||||
| **Stream** | `SmartArchive.fromArchiveStream(stream)` | Process archives from any Node.js stream |
|
||||
```typescript
|
||||
SmartArchive.create() // Start a new builder
|
||||
.source(...) // Configure source (extraction mode)
|
||||
.options(...) // Set options
|
||||
.terminal() // Execute the operation
|
||||
```
|
||||
|
||||
### Export Destinations
|
||||
### Two Operating Modes
|
||||
|
||||
| Destination | Method | Use Case |
|
||||
|-------------|--------|----------|
|
||||
| **Filesystem** | `exportToFs(targetDir, fileName?)` | Extract directly to a directory |
|
||||
| **Stream of files** | `exportToStreamOfStreamFiles()` | Process files individually as `StreamFile` objects |
|
||||
**Extraction Mode** - Load an existing archive and extract/analyze it:
|
||||
```typescript
|
||||
SmartArchive.create()
|
||||
.url('...') // or .file(), .stream(), .buffer()
|
||||
.extract('./out') // or .toSmartFiles(), .list(), etc.
|
||||
```
|
||||
|
||||
**Creation Mode** - Build a new archive from entries:
|
||||
```typescript
|
||||
SmartArchive.create()
|
||||
.format('tar.gz') // Set output format
|
||||
.entry(...) // Add files
|
||||
.toFile('./out.tar.gz') // or .toBuffer(), .toStream()
|
||||
```
|
||||
|
||||
> ⚠️ **Note:** You cannot mix extraction and creation methods in the same chain.
|
||||
|
||||
## API Reference 📚
|
||||
|
||||
### Source Methods (Extraction Mode)
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `.url(url)` | Load archive from a URL |
|
||||
| `.file(path)` | Load archive from local filesystem |
|
||||
| `.stream(readable)` | Load archive from any Node.js readable stream |
|
||||
| `.buffer(buffer)` | Load archive from an in-memory Buffer |
|
||||
|
||||
### Creation Methods (Creation Mode)
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `.format(fmt)` | Set output format: `'tar'`, `'tar.gz'`, `'tgz'`, `'zip'`, `'gz'` |
|
||||
| `.compression(level)` | Set compression level (0-9, default: 6) |
|
||||
| `.entry(path, content)` | Add a file entry (string or Buffer content) |
|
||||
| `.entries(array)` | Add multiple entries at once |
|
||||
| `.directory(path, archiveBase?)` | Add entire directory contents |
|
||||
| `.addSmartFile(file, path?)` | Add a SmartFile instance |
|
||||
| `.addStreamFile(file, path?)` | Add a StreamFile instance |
|
||||
|
||||
### Filter Methods (Both Modes)
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `.filter(predicate)` | Filter entries with custom function |
|
||||
| `.include(pattern)` | Only include entries matching regex/string pattern |
|
||||
| `.exclude(pattern)` | Exclude entries matching regex/string pattern |
|
||||
|
||||
### Extraction Options
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `.stripComponents(n)` | Strip N leading path components |
|
||||
| `.overwrite(bool)` | Overwrite existing files (default: false) |
|
||||
| `.fileName(name)` | Set output filename for single-file archives (gz, bz2) |
|
||||
|
||||
### Terminal Methods (Extraction)
|
||||
|
||||
| Method | Returns | Description |
|
||||
|--------|---------|-------------|
|
||||
| `.extract(targetDir)` | `Promise<void>` | Extract to filesystem directory |
|
||||
| `.toStreamFiles()` | `Promise<StreamIntake<StreamFile>>` | Get stream of StreamFile objects |
|
||||
| `.toSmartFiles()` | `Promise<SmartFile[]>` | Get in-memory SmartFile array |
|
||||
| `.extractFile(path)` | `Promise<SmartFile \| null>` | Extract single file by path |
|
||||
| `.list()` | `Promise<IArchiveEntryInfo[]>` | List all entries |
|
||||
| `.analyze()` | `Promise<IArchiveInfo>` | Get archive metadata |
|
||||
| `.hasFile(path)` | `Promise<boolean>` | Check if file exists |
|
||||
|
||||
### Terminal Methods (Creation)
|
||||
|
||||
| Method | Returns | Description |
|
||||
|--------|---------|-------------|
|
||||
| `.build()` | `Promise<SmartArchive>` | Build the archive (implicit in other terminals) |
|
||||
| `.toBuffer()` | `Promise<Buffer>` | Get archive as Buffer |
|
||||
| `.toFile(path)` | `Promise<void>` | Write archive to disk |
|
||||
| `.toStream()` | `Promise<Readable>` | Get raw archive stream |
|
||||
|
||||
## Usage Examples 🔨
|
||||
|
||||
### Working with ZIP files
|
||||
### Download and extract npm packages
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Extract a ZIP file
|
||||
const zipArchive = await SmartArchive.fromArchiveFile('./archive.zip');
|
||||
await zipArchive.exportToFs('./output');
|
||||
const pkg = await SmartArchive.create()
|
||||
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz');
|
||||
|
||||
// Stream ZIP contents for processing
|
||||
const fileStream = await zipArchive.exportToStreamOfStreamFiles();
|
||||
// Quick inspection of package.json
|
||||
const pkgJson = await pkg.extractFile('package/package.json');
|
||||
if (pkgJson) {
|
||||
const metadata = JSON.parse(pkgJson.contents.toString());
|
||||
console.log(`Package: ${metadata.name}@${metadata.version}`);
|
||||
}
|
||||
|
||||
// Full extraction with path normalization
|
||||
await SmartArchive.create()
|
||||
.url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||
.stripComponents(1)
|
||||
.extract('./node_modules/lodash');
|
||||
```
|
||||
|
||||
### Create ZIP archive
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
await SmartArchive.create()
|
||||
.format('zip')
|
||||
.compression(9)
|
||||
.entry('report.txt', 'Monthly sales report...')
|
||||
.entry('data/figures.json', JSON.stringify({ revenue: 10000 }))
|
||||
.entry('images/logo.png', pngBuffer)
|
||||
.toFile('./report-bundle.zip');
|
||||
```
|
||||
|
||||
### Create TAR.GZ from directory
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
await SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.compression(9)
|
||||
.directory('./src', 'source') // Archive ./src as 'source/' in archive
|
||||
.toFile('./project-backup.tar.gz');
|
||||
```
|
||||
|
||||
### Stream-based extraction
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
const fileStream = await SmartArchive.create()
|
||||
.file('./large-archive.tar.gz')
|
||||
.toStreamFiles();
|
||||
|
||||
fileStream.on('data', async (streamFile) => {
|
||||
console.log(`Processing: ${streamFile.relativeFilePath}`);
|
||||
|
||||
if (streamFile.relativeFilePath.endsWith('.json')) {
|
||||
const readStream = await streamFile.createReadStream();
|
||||
// Process JSON files from the archive
|
||||
const content = await streamFile.getContentAsBuffer();
|
||||
const data = JSON.parse(content.toString());
|
||||
// Process JSON data...
|
||||
}
|
||||
});
|
||||
|
||||
fileStream.on('end', () => {
|
||||
console.log('Extraction complete');
|
||||
});
|
||||
```
|
||||
|
||||
### Working with TAR archives
|
||||
### Filter specific file types
|
||||
|
||||
```typescript
|
||||
import { SmartArchive, TarTools } from '@push.rocks/smartarchive';
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Extract a .tar.gz file
|
||||
const tarGzArchive = await SmartArchive.fromArchiveFile('./archive.tar.gz');
|
||||
await tarGzArchive.exportToFs('./extracted');
|
||||
// Extract only TypeScript files
|
||||
const tsFiles = await SmartArchive.create()
|
||||
.url('https://example.com/project.tar.gz')
|
||||
.include(/\.ts$/)
|
||||
.exclude(/node_modules/)
|
||||
.toSmartFiles();
|
||||
|
||||
for (const file of tsFiles) {
|
||||
console.log(`${file.relative}: ${file.contents.length} bytes`);
|
||||
}
|
||||
```
|
||||
|
||||
### Analyze archive without extraction
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
const archive = SmartArchive.create()
|
||||
.file('./unknown-archive.tar.gz');
|
||||
|
||||
// Get format info
|
||||
const info = await archive.analyze();
|
||||
console.log(`Format: ${info.format}`);
|
||||
console.log(`Compressed: ${info.isCompressed}`);
|
||||
|
||||
// List contents
|
||||
const entries = await archive.list();
|
||||
for (const entry of entries) {
|
||||
console.log(`${entry.path} (${entry.isDirectory ? 'dir' : 'file'})`);
|
||||
}
|
||||
|
||||
// Check for specific file
|
||||
if (await archive.hasFile('package.json')) {
|
||||
const pkgFile = await archive.extractFile('package.json');
|
||||
console.log(pkgFile?.contents.toString());
|
||||
}
|
||||
```
|
||||
|
||||
### Working with GZIP files
|
||||
|
||||
```typescript
|
||||
import { SmartArchive, GzipTools } from '@push.rocks/smartarchive';
|
||||
|
||||
// Decompress a .gz file
|
||||
await SmartArchive.create()
|
||||
.file('./data.json.gz')
|
||||
.fileName('data.json') // Specify output name (gzip doesn't store filename)
|
||||
.extract('./decompressed');
|
||||
|
||||
// Use GzipTools directly for compression/decompression
|
||||
const gzipTools = new GzipTools();
|
||||
|
||||
// Compress a buffer
|
||||
const compressed = await gzipTools.compress(Buffer.from('Hello World'), 9);
|
||||
const decompressed = await gzipTools.decompress(compressed);
|
||||
|
||||
// Synchronous operations
|
||||
const compressedSync = gzipTools.compressSync(inputBuffer, 6);
|
||||
const decompressedSync = gzipTools.decompressSync(compressedSync);
|
||||
|
||||
// Streaming
|
||||
const compressStream = gzipTools.getCompressionStream(6);
|
||||
const decompressStream = gzipTools.getDecompressionStream();
|
||||
|
||||
createReadStream('./input.txt')
|
||||
.pipe(compressStream)
|
||||
.pipe(createWriteStream('./output.gz'));
|
||||
```
|
||||
|
||||
### Working with TAR archives directly
|
||||
|
||||
```typescript
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
|
||||
// Create a TAR archive using TarTools directly
|
||||
const tarTools = new TarTools();
|
||||
|
||||
// Create a TAR archive manually
|
||||
const pack = await tarTools.getPackStream();
|
||||
|
||||
// Add files to the pack
|
||||
await tarTools.addFileToPack(pack, {
|
||||
fileName: 'hello.txt',
|
||||
content: 'Hello, World!'
|
||||
@@ -131,262 +331,56 @@ await tarTools.addFileToPack(pack, {
|
||||
content: Buffer.from(JSON.stringify({ foo: 'bar' }))
|
||||
});
|
||||
|
||||
// Finalize and pipe to destination
|
||||
pack.finalize();
|
||||
pack.pipe(createWriteStream('./output.tar'));
|
||||
|
||||
// Pack a directory to TAR.GZ buffer
|
||||
const tgzBuffer = await tarTools.packDirectoryToTarGz('./src', 6);
|
||||
|
||||
// Pack a directory to TAR.GZ stream
|
||||
const tgzStream = await tarTools.packDirectoryToTarGzStream('./src');
|
||||
```
|
||||
|
||||
### Pack a directory into TAR
|
||||
|
||||
```typescript
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
import { createWriteStream } from 'fs';
|
||||
|
||||
const tarTools = new TarTools();
|
||||
|
||||
// Pack an entire directory
|
||||
const pack = await tarTools.packDirectory('./src');
|
||||
pack.finalize();
|
||||
pack.pipe(createWriteStream('./source.tar'));
|
||||
```
|
||||
|
||||
### Extracting from URLs
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Download and extract npm packages
|
||||
const npmPackage = await SmartArchive.fromArchiveUrl(
|
||||
'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz'
|
||||
);
|
||||
await npmPackage.exportToFs('./node_modules/@push.rocks/smartfile');
|
||||
|
||||
// Or process as stream for memory efficiency
|
||||
const stream = await npmPackage.exportToStreamOfStreamFiles();
|
||||
stream.on('data', async (file) => {
|
||||
console.log(`Extracted: ${file.relativeFilePath}`);
|
||||
});
|
||||
```
|
||||
|
||||
### Working with GZIP files
|
||||
|
||||
```typescript
|
||||
import { SmartArchive, GzipTools } from '@push.rocks/smartarchive';
|
||||
import { createReadStream, createWriteStream } from 'fs';
|
||||
|
||||
// Decompress a .gz file - provide filename since gzip doesn't store it
|
||||
const gzipArchive = await SmartArchive.fromArchiveFile('./data.json.gz');
|
||||
await gzipArchive.exportToFs('./decompressed', 'data.json');
|
||||
|
||||
// Use GzipTools directly for streaming decompression
|
||||
const gzipTools = new GzipTools();
|
||||
const decompressStream = gzipTools.getDecompressionStream();
|
||||
|
||||
createReadStream('./compressed.gz')
|
||||
.pipe(decompressStream)
|
||||
.pipe(createWriteStream('./decompressed.txt'));
|
||||
```
|
||||
|
||||
### Working with BZIP2 files
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Handle .bz2 files
|
||||
const bzipArchive = await SmartArchive.fromArchiveUrl(
|
||||
'https://example.com/data.bz2'
|
||||
);
|
||||
await bzipArchive.exportToFs('./extracted', 'data.txt');
|
||||
```
|
||||
|
||||
### In-memory processing (no filesystem)
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
// Process archives entirely in memory
|
||||
const compressedBuffer = await fetchCompressedData();
|
||||
const memoryStream = Readable.from(compressedBuffer);
|
||||
|
||||
const archive = await SmartArchive.fromArchiveStream(memoryStream);
|
||||
const streamFiles = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
const extractedFiles: Array<{ name: string; content: Buffer }> = [];
|
||||
|
||||
streamFiles.on('data', async (streamFile) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await streamFile.createReadStream();
|
||||
|
||||
for await (const chunk of readStream) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
extractedFiles.push({
|
||||
name: streamFile.relativeFilePath,
|
||||
content: Buffer.concat(chunks)
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise((resolve) => streamFiles.on('end', resolve));
|
||||
console.log(`Extracted ${extractedFiles.length} files in memory`);
|
||||
```
|
||||
|
||||
### Nested archive handling (e.g., .tar.gz)
|
||||
|
||||
The library automatically handles nested compression. A `.tar.gz` file is:
|
||||
1. First decompressed from gzip
|
||||
2. Then unpacked from tar
|
||||
|
||||
This happens transparently:
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Automatically handles gzip → tar extraction chain
|
||||
const tgzArchive = await SmartArchive.fromArchiveFile('./package.tar.gz');
|
||||
await tgzArchive.exportToFs('./extracted');
|
||||
```
|
||||
|
||||
## API Reference 📚
|
||||
|
||||
### SmartArchive Class
|
||||
|
||||
The main entry point for archive operations.
|
||||
|
||||
#### Static Factory Methods
|
||||
|
||||
```typescript
|
||||
// Create from URL - downloads and processes archive
|
||||
SmartArchive.fromArchiveUrl(url: string): Promise<SmartArchive>
|
||||
|
||||
// Create from local file path
|
||||
SmartArchive.fromArchiveFile(path: string): Promise<SmartArchive>
|
||||
|
||||
// Create from any Node.js readable stream
|
||||
SmartArchive.fromArchiveStream(stream: Readable | Duplex | Transform): Promise<SmartArchive>
|
||||
```
|
||||
|
||||
#### Instance Methods
|
||||
|
||||
```typescript
|
||||
// Extract all files to a directory
|
||||
// fileName is optional - used for single-file archives (like .gz) that don't store filename
|
||||
exportToFs(targetDir: string, fileName?: string): Promise<void>
|
||||
|
||||
// Get a stream that emits StreamFile objects for each file in the archive
|
||||
exportToStreamOfStreamFiles(): Promise<StreamIntake<StreamFile>>
|
||||
|
||||
// Get the raw archive stream (useful for piping)
|
||||
getArchiveStream(): Promise<Readable>
|
||||
```
|
||||
|
||||
#### Instance Properties
|
||||
|
||||
```typescript
|
||||
archive.tarTools // TarTools instance for TAR-specific operations
|
||||
archive.zipTools // ZipTools instance for ZIP-specific operations
|
||||
archive.gzipTools // GzipTools instance for GZIP-specific operations
|
||||
archive.bzip2Tools // Bzip2Tools instance for BZIP2-specific operations
|
||||
archive.archiveAnalyzer // ArchiveAnalyzer for inspecting archive type
|
||||
```
|
||||
|
||||
### TarTools Class
|
||||
|
||||
TAR-specific operations for creating and extracting TAR archives.
|
||||
|
||||
```typescript
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const tarTools = new TarTools();
|
||||
|
||||
// Get a tar pack stream for creating archives
|
||||
const pack = await tarTools.getPackStream();
|
||||
|
||||
// Add files to a pack stream
|
||||
await tarTools.addFileToPack(pack, {
|
||||
fileName: 'file.txt', // Name in archive
|
||||
content: 'Hello World', // String, Buffer, Readable, SmartFile, or StreamFile
|
||||
byteLength?: number, // Optional: specify size for streams
|
||||
filePath?: string // Optional: path to file on disk
|
||||
});
|
||||
|
||||
// Pack an entire directory
|
||||
const pack = await tarTools.packDirectory('./src');
|
||||
|
||||
// Get extraction stream
|
||||
const extract = tarTools.getDecompressionStream();
|
||||
```
|
||||
|
||||
### ZipTools Class
|
||||
|
||||
ZIP-specific operations.
|
||||
### Working with ZIP archives directly
|
||||
|
||||
```typescript
|
||||
import { ZipTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const zipTools = new ZipTools();
|
||||
|
||||
// Get compression stream (for creating ZIP)
|
||||
const compressor = zipTools.getCompressionStream();
|
||||
// Create a ZIP archive from entries
|
||||
const zipBuffer = await zipTools.createZip([
|
||||
{ archivePath: 'readme.txt', content: 'Hello!' },
|
||||
{ archivePath: 'data.bin', content: Buffer.from([0x00, 0x01, 0x02]) }
|
||||
], 6);
|
||||
|
||||
// Get decompression stream (for extracting ZIP)
|
||||
const decompressor = zipTools.getDecompressionStream();
|
||||
// Extract a ZIP buffer
|
||||
const entries = await zipTools.extractZip(zipBuffer);
|
||||
for (const entry of entries) {
|
||||
console.log(`${entry.path}: ${entry.content.length} bytes`);
|
||||
}
|
||||
```
|
||||
|
||||
### GzipTools Class
|
||||
|
||||
GZIP compression/decompression streams.
|
||||
|
||||
```typescript
|
||||
import { GzipTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const gzipTools = new GzipTools();
|
||||
|
||||
// Get compression stream
|
||||
const compressor = gzipTools.getCompressionStream();
|
||||
|
||||
// Get decompression stream
|
||||
const decompressor = gzipTools.getDecompressionStream();
|
||||
```
|
||||
|
||||
## Supported Formats 📋
|
||||
|
||||
| Format | Extension(s) | Extract | Create |
|
||||
|--------|--------------|---------|--------|
|
||||
| TAR | `.tar` | ✅ | ✅ |
|
||||
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ⚠️ |
|
||||
| ZIP | `.zip` | ✅ | ⚠️ |
|
||||
| GZIP | `.gz` | ✅ | ✅ |
|
||||
| BZIP2 | `.bz2` | ✅ | ❌ |
|
||||
|
||||
✅ Full support | ⚠️ Partial/basic support | ❌ Not supported
|
||||
|
||||
## Performance Tips 🏎️
|
||||
|
||||
1. **Use streaming for large files** – Avoid loading entire archives into memory with `exportToStreamOfStreamFiles()`
|
||||
2. **Provide byte lengths when known** – When adding streams to TAR, provide `byteLength` for better performance
|
||||
3. **Process files as they stream** – Don't collect all files into an array unless necessary
|
||||
4. **Choose the right format** – TAR.GZ for Unix/compression, ZIP for cross-platform compatibility
|
||||
|
||||
## Error Handling 🛡️
|
||||
### In-memory round-trip
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
try {
|
||||
const archive = await SmartArchive.fromArchiveUrl('https://example.com/file.zip');
|
||||
await archive.exportToFs('./output');
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.error('Archive file not found');
|
||||
} else if (error.code === 'EACCES') {
|
||||
console.error('Permission denied');
|
||||
} else if (error.message.includes('fetch')) {
|
||||
console.error('Network error downloading archive');
|
||||
} else {
|
||||
console.error('Archive extraction failed:', error.message);
|
||||
}
|
||||
// Create archive in memory
|
||||
const archive = await SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.entry('config.json', JSON.stringify({ version: '1.0.0' }))
|
||||
.build();
|
||||
|
||||
const buffer = await archive.toBuffer();
|
||||
|
||||
// Extract from buffer
|
||||
const files = await SmartArchive.create()
|
||||
.buffer(buffer)
|
||||
.toSmartFiles();
|
||||
|
||||
for (const file of files) {
|
||||
console.log(`${file.relative}: ${file.contents.toString()}`);
|
||||
}
|
||||
```
|
||||
|
||||
@@ -395,51 +389,139 @@ try {
|
||||
### CI/CD: Download & Extract Build Artifacts
|
||||
|
||||
```typescript
|
||||
const artifacts = await SmartArchive.fromArchiveUrl(
|
||||
`${CI_SERVER}/artifacts/build-${BUILD_ID}.zip`
|
||||
);
|
||||
await artifacts.exportToFs('./dist');
|
||||
const artifacts = await SmartArchive.create()
|
||||
.url(`${CI_SERVER}/artifacts/build-${BUILD_ID}.zip`)
|
||||
.stripComponents(1)
|
||||
.extract('./dist');
|
||||
```
|
||||
|
||||
### Backup System: Restore from Archive
|
||||
### Backup System
|
||||
|
||||
```typescript
|
||||
const backup = await SmartArchive.fromArchiveFile('./backup-2024.tar.gz');
|
||||
await backup.exportToFs('/restore/location');
|
||||
// Create backup
|
||||
await SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.compression(9)
|
||||
.directory('./data')
|
||||
.toFile(`./backups/backup-${Date.now()}.tar.gz`);
|
||||
|
||||
// Restore backup
|
||||
await SmartArchive.create()
|
||||
.file('./backups/backup-latest.tar.gz')
|
||||
.extract('/restore/location');
|
||||
```
|
||||
|
||||
### NPM Package Inspection
|
||||
### Bundle files for HTTP download
|
||||
|
||||
```typescript
|
||||
const pkg = await SmartArchive.fromArchiveUrl(
|
||||
'https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz'
|
||||
);
|
||||
const files = await pkg.exportToStreamOfStreamFiles();
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
files.on('data', async (file) => {
|
||||
if (file.relativeFilePath.includes('package.json')) {
|
||||
const stream = await file.createReadStream();
|
||||
// Read and analyze package.json
|
||||
}
|
||||
// Express/Fastify handler
|
||||
app.get('/download-bundle', async (req, res) => {
|
||||
const buffer = await SmartArchive.create()
|
||||
.format('zip')
|
||||
.entry('report.pdf', pdfBuffer)
|
||||
.entry('data.xlsx', excelBuffer)
|
||||
.entry('images/chart.png', chartBuffer)
|
||||
.toBuffer();
|
||||
|
||||
res.setHeader('Content-Type', 'application/zip');
|
||||
res.setHeader('Content-Disposition', 'attachment; filename=report-bundle.zip');
|
||||
res.send(buffer);
|
||||
});
|
||||
```
|
||||
|
||||
### Data Pipeline: Process Compressed Datasets
|
||||
|
||||
```typescript
|
||||
const dataset = await SmartArchive.fromArchiveUrl(
|
||||
'https://data.source/dataset.tar.gz'
|
||||
);
|
||||
const fileStream = await SmartArchive.create()
|
||||
.url('https://data.source/dataset.tar.gz')
|
||||
.toStreamFiles();
|
||||
|
||||
const files = await dataset.exportToStreamOfStreamFiles();
|
||||
files.on('data', async (file) => {
|
||||
fileStream.on('data', async (file) => {
|
||||
if (file.relativeFilePath.endsWith('.csv')) {
|
||||
const stream = await file.createReadStream();
|
||||
// Stream CSV processing
|
||||
const content = await file.getContentAsBuffer();
|
||||
// Stream CSV processing...
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Supported Formats 📋
|
||||
|
||||
| Format | Extension(s) | Extract | Create |
|
||||
|--------|--------------|---------|--------|
|
||||
| TAR | `.tar` | ✅ | ✅ |
|
||||
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ✅ |
|
||||
| ZIP | `.zip` | ✅ | ✅ |
|
||||
| GZIP | `.gz` | ✅ | ✅ |
|
||||
| BZIP2 | `.bz2` | ✅ | ❌ |
|
||||
|
||||
## Type Definitions
|
||||
|
||||
```typescript
|
||||
// Supported archive formats
|
||||
type TArchiveFormat = 'tar' | 'tar.gz' | 'tgz' | 'zip' | 'gz' | 'bz2';
|
||||
|
||||
// Compression level (0 = none, 9 = maximum)
|
||||
type TCompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
|
||||
|
||||
// Entry for creating archives
|
||||
interface IArchiveEntry {
|
||||
archivePath: string;
|
||||
content: string | Buffer | Readable | SmartFile | StreamFile;
|
||||
size?: number;
|
||||
mode?: number;
|
||||
mtime?: Date;
|
||||
}
|
||||
|
||||
// Information about an archive entry
|
||||
interface IArchiveEntryInfo {
|
||||
path: string;
|
||||
size: number;
|
||||
isDirectory: boolean;
|
||||
isFile: boolean;
|
||||
mtime?: Date;
|
||||
mode?: number;
|
||||
}
|
||||
|
||||
// Archive analysis result
|
||||
interface IArchiveInfo {
|
||||
format: TArchiveFormat | null;
|
||||
isCompressed: boolean;
|
||||
isArchive: boolean;
|
||||
entries?: IArchiveEntryInfo[];
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Tips 🏎️
|
||||
|
||||
1. **Use streaming for large files** – `.toStreamFiles()` processes entries one at a time without loading the entire archive
|
||||
2. **Provide byte lengths when known** – When using TarTools directly, provide `byteLength` for better performance
|
||||
3. **Choose appropriate compression** – Use 1-3 for speed, 6 (default) for balance, 9 for maximum compression
|
||||
4. **Filter early** – Use `.include()`/`.exclude()` to skip unwanted entries before processing
|
||||
|
||||
## Error Handling 🛡️
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
try {
|
||||
await SmartArchive.create()
|
||||
.url('https://example.com/file.zip')
|
||||
.extract('./output');
|
||||
} catch (error) {
|
||||
if (error.message.includes('No source configured')) {
|
||||
console.error('Forgot to specify source');
|
||||
} else if (error.message.includes('No format specified')) {
|
||||
console.error('Forgot to set format for creation');
|
||||
} else if (error.message.includes('extraction mode')) {
|
||||
console.error('Cannot mix extraction and creation methods');
|
||||
} else {
|
||||
console.error('Archive operation failed:', error.message);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
@@ -450,6 +532,10 @@ This repository contains open-source code that is licensed under the MIT License
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
|
||||
|
||||
### Issue Reporting and Security
|
||||
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
|
||||
@@ -37,16 +37,14 @@ tap.test('should create and extract a gzip file', async () => {
|
||||
Buffer.from(compressed)
|
||||
);
|
||||
|
||||
// Now test extraction using SmartArchive
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Export to a new location
|
||||
// Now test extraction using SmartArchive fluent API
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'extracted');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.extractToDirectory(extractPath, { fileName: 'test-file.txt' });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||
.fileName('test-file.txt')
|
||||
.extract(extractPath);
|
||||
|
||||
// Read the extracted file
|
||||
const extractedContent = await plugins.fsPromises.readFile(
|
||||
@@ -76,12 +74,11 @@ tap.test('should handle gzip stream extraction', async () => {
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction using SmartArchive from stream
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromStream(gzipStream);
|
||||
|
||||
// Export to stream and collect the result
|
||||
// Test extraction using SmartArchive from stream with fluent API
|
||||
const streamFiles: any[] = [];
|
||||
const resultStream = await gzipArchive.extractToStream();
|
||||
const resultStream = await smartarchive.SmartArchive.create()
|
||||
.stream(gzipStream)
|
||||
.toStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
resultStream.on('data', (streamFile) => {
|
||||
@@ -112,7 +109,6 @@ tap.test('should handle gzip stream extraction', async () => {
|
||||
tap.test('should handle gzip files with original filename in header', async () => {
|
||||
// Test with a real-world gzip file that includes filename in header
|
||||
const testContent = 'File with name in gzip header\n'.repeat(30);
|
||||
const originalFileName = 'original-name.log';
|
||||
const gzipFileName = 'compressed.gz';
|
||||
|
||||
// Create a proper gzip with filename header using Node's zlib
|
||||
@@ -120,8 +116,6 @@ tap.test('should handle gzip files with original filename in header', async () =
|
||||
const gzipBuffer = await new Promise<Buffer>((resolve, reject) => {
|
||||
zlib.gzip(Buffer.from(testContent), {
|
||||
level: 9,
|
||||
// Note: Node's zlib doesn't support embedding filename directly,
|
||||
// but we can test the extraction anyway
|
||||
}, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(result);
|
||||
@@ -133,15 +127,14 @@ tap.test('should handle gzip files with original filename in header', async () =
|
||||
gzipBuffer
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction with fluent API
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'header-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't reliably contain filename metadata
|
||||
await gzipArchive.extractToDirectory(extractPath, { fileName: 'compressed.txt' });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||
.fileName('compressed.txt')
|
||||
.extract(extractPath);
|
||||
|
||||
// Check if file was extracted (name might be derived from archive name)
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
@@ -169,15 +162,14 @@ tap.test('should handle large gzip files', async () => {
|
||||
Buffer.from(compressed)
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction with fluent API
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'large-extracted');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.extractToDirectory(extractPath, { fileName: 'large-file.txt' });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
|
||||
.fileName('large-file.txt')
|
||||
.extract(extractPath);
|
||||
|
||||
// Verify the extracted content
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
@@ -195,14 +187,13 @@ tap.test('should handle real-world multi-chunk gzip from URL', async () => {
|
||||
// Test with a real tgz file that will be processed in multiple chunks
|
||||
const testUrl = 'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz';
|
||||
|
||||
// Download and extract the archive
|
||||
const testArchive = await smartarchive.SmartArchive.fromUrl(testUrl);
|
||||
|
||||
// Download and extract the archive with fluent API
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'real-world-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
|
||||
// This will test multi-chunk decompression as the file is larger
|
||||
await testArchive.extractToDirectory(extractPath);
|
||||
await smartarchive.SmartArchive.create()
|
||||
.url(testUrl)
|
||||
.extract(extractPath);
|
||||
|
||||
// Verify extraction worked
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
@@ -270,16 +261,11 @@ tap.test('should handle gzip extraction fully in memory', async () => {
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
|
||||
// Create a stream from the compressed data
|
||||
const { Readable } = await import('node:stream');
|
||||
const compressedStream = Readable.from(Buffer.from(compressed));
|
||||
|
||||
// Process through SmartArchive without touching filesystem
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromStream(compressedStream);
|
||||
|
||||
// Export to stream of stream files (in memory)
|
||||
// Process through SmartArchive without touching filesystem using fluent API
|
||||
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||
const resultStream = await gzipArchive.extractToStream();
|
||||
const resultStream = await smartarchive.SmartArchive.create()
|
||||
.buffer(Buffer.from(compressed))
|
||||
.toStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||
@@ -318,16 +304,11 @@ tap.test('should handle real tgz file fully in memory', async (tools) => {
|
||||
const tgzBuffer = Buffer.from(await response.arrayBuffer());
|
||||
console.log(` Downloaded ${tgzBuffer.length} bytes into memory`);
|
||||
|
||||
// Create stream from buffer
|
||||
const { Readable: Readable2 } = await import('node:stream');
|
||||
const tgzStream = Readable2.from(tgzBuffer);
|
||||
|
||||
// Process through SmartArchive in memory
|
||||
const archive = await smartarchive.SmartArchive.fromStream(tgzStream);
|
||||
|
||||
// Export to stream of stream files (in memory)
|
||||
// Process through SmartArchive in memory with fluent API
|
||||
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||
const resultStream = await archive.extractToStream();
|
||||
const resultStream = await smartarchive.SmartArchive.create()
|
||||
.buffer(tgzBuffer)
|
||||
.toStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timeout: NodeJS.Timeout;
|
||||
|
||||
@@ -32,20 +32,209 @@ tap.preTask('should prepare downloads', async (tools) => {
|
||||
);
|
||||
});
|
||||
|
||||
tap.test('should extract existing files on disk', async () => {
|
||||
const testSmartarchive = await smartarchive.SmartArchive.fromUrl(
|
||||
'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz',
|
||||
);
|
||||
await testSmartarchive.extractToDirectory(testPaths.nogitDir);
|
||||
tap.test('should extract existing files on disk using fluent API', async () => {
|
||||
await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.extract(testPaths.nogitDir);
|
||||
});
|
||||
|
||||
tap.test('should extract from file using fluent API', async () => {
|
||||
const extractPath = plugins.path.join(testPaths.nogitDir, 'from-file-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
|
||||
.extract(extractPath);
|
||||
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('should extract with stripComponents using fluent API', async () => {
|
||||
const extractPath = plugins.path.join(testPaths.nogitDir, 'strip-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.stripComponents(1)
|
||||
.extract(extractPath);
|
||||
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
// Files should not have 'package/' prefix
|
||||
const hasPackagePrefix = files.some(f => f.startsWith('package/'));
|
||||
expect(hasPackagePrefix).toBeFalse();
|
||||
});
|
||||
|
||||
tap.test('should extract with filter using fluent API', async () => {
|
||||
const extractPath = plugins.path.join(testPaths.nogitDir, 'filter-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.filter(entry => entry.path.endsWith('.json'))
|
||||
.extract(extractPath);
|
||||
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
// All extracted files should be JSON
|
||||
for (const file of files) {
|
||||
expect(file.endsWith('.json')).toBeTrue();
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should list archive entries using fluent API', async () => {
|
||||
const entries = await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.list();
|
||||
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
const hasPackageJson = entries.some(e => e.path.includes('package.json'));
|
||||
expect(hasPackageJson).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('should create archive using fluent API', async () => {
|
||||
const archive = await smartarchive.SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.compression(9)
|
||||
.entry('hello.txt', 'Hello World!')
|
||||
.entry('config.json', JSON.stringify({ name: 'test', version: '1.0.0' }));
|
||||
|
||||
expect(archive).toBeInstanceOf(smartarchive.SmartArchive);
|
||||
|
||||
const buffer = await archive.toBuffer();
|
||||
expect(buffer.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('should create and write archive to file using fluent API', async () => {
|
||||
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.tar.gz');
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.format('tar.gz')
|
||||
.entry('readme.txt', 'This is a test archive')
|
||||
.entry('data/info.json', JSON.stringify({ created: new Date().toISOString() }))
|
||||
.toFile(outputPath);
|
||||
|
||||
// Verify file was created
|
||||
const stats = await plugins.fsPromises.stat(outputPath);
|
||||
expect(stats.size).toBeGreaterThan(0);
|
||||
|
||||
// Verify we can extract it
|
||||
const extractPath = plugins.path.join(testPaths.nogitDir, 'verify-created');
|
||||
await smartarchive.SmartArchive.create()
|
||||
.file(outputPath)
|
||||
.extract(extractPath);
|
||||
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files).toContain('readme.txt');
|
||||
});
|
||||
|
||||
tap.test('should create ZIP archive using fluent API', async () => {
|
||||
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.zip');
|
||||
|
||||
await smartarchive.SmartArchive.create()
|
||||
.format('zip')
|
||||
.entry('file1.txt', 'Content 1')
|
||||
.entry('file2.txt', 'Content 2')
|
||||
.toFile(outputPath);
|
||||
|
||||
// Verify file was created
|
||||
const stats = await plugins.fsPromises.stat(outputPath);
|
||||
expect(stats.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
tap.test('should extract to SmartFiles using fluent API', async () => {
|
||||
const smartFiles = await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.toSmartFiles();
|
||||
|
||||
expect(smartFiles.length).toBeGreaterThan(0);
|
||||
|
||||
const packageJson = smartFiles.find(f => f.relative.includes('package.json'));
|
||||
expect(packageJson).toBeDefined();
|
||||
});
|
||||
|
||||
tap.test('should analyze archive using fluent API', async () => {
|
||||
const info = await smartarchive.SmartArchive.create()
|
||||
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
|
||||
.analyze();
|
||||
|
||||
expect(info.isArchive).toBeTrue();
|
||||
expect(info.isCompressed).toBeTrue();
|
||||
expect(info.format).toEqual('gz');
|
||||
});
|
||||
|
||||
tap.test('should check if file exists in archive using fluent API', async () => {
|
||||
const hasPackageJson = await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.hasFile('package.json');
|
||||
|
||||
expect(hasPackageJson).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('should extract single file using fluent API', async () => {
|
||||
const packageJson = await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.extractFile('package.json');
|
||||
|
||||
expect(packageJson).toBeDefined();
|
||||
expect(packageJson!.contents.toString()).toContain('websetup');
|
||||
});
|
||||
|
||||
tap.test('should handle include/exclude patterns', async () => {
|
||||
const smartFiles = await smartarchive.SmartArchive.create()
|
||||
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
|
||||
.include(/\.json$/)
|
||||
.toSmartFiles();
|
||||
|
||||
expect(smartFiles.length).toBeGreaterThan(0);
|
||||
for (const file of smartFiles) {
|
||||
expect(file.relative.endsWith('.json')).toBeTrue();
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should throw error when mixing modes', async () => {
|
||||
let threw = false;
|
||||
try {
|
||||
smartarchive.SmartArchive.create()
|
||||
.url('https://example.com/archive.tgz')
|
||||
.entry('file.txt', 'content'); // This should throw
|
||||
} catch (e) {
|
||||
threw = true;
|
||||
expect((e as Error).message).toContain('extraction mode');
|
||||
}
|
||||
expect(threw).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('should throw error when no source configured', async () => {
|
||||
let threw = false;
|
||||
try {
|
||||
await smartarchive.SmartArchive.create().extract('./output');
|
||||
} catch (e) {
|
||||
threw = true;
|
||||
expect((e as Error).message).toContain('No source configured');
|
||||
}
|
||||
expect(threw).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('should throw error when no format configured', async () => {
|
||||
let threw = false;
|
||||
try {
|
||||
await smartarchive.SmartArchive.create()
|
||||
.entry('file.txt', 'content')
|
||||
.toBuffer();
|
||||
} catch (e) {
|
||||
threw = true;
|
||||
expect((e as Error).message).toContain('No format specified');
|
||||
}
|
||||
expect(threw).toBeTrue();
|
||||
});
|
||||
|
||||
tap.skip.test('should extract a b2zip', async () => {
|
||||
const dataUrl =
|
||||
'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||||
const testArchive = await smartarchive.SmartArchive.fromUrl(dataUrl);
|
||||
await testArchive.extractToDirectory(
|
||||
plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'),
|
||||
);
|
||||
await smartarchive.SmartArchive.create()
|
||||
.url(dataUrl)
|
||||
.extract(plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'));
|
||||
});
|
||||
|
||||
await tap.start();
|
||||
export default tap.start();
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartarchive',
|
||||
version: '5.0.0',
|
||||
version: '5.0.1',
|
||||
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
|
||||
}
|
||||
|
||||
@@ -118,26 +118,21 @@ export class GzipTools {
|
||||
|
||||
/**
|
||||
* Compress data asynchronously
|
||||
* Note: Uses sync version for Deno compatibility (fflate async uses Web Workers
|
||||
* which have issues in Deno)
|
||||
*/
|
||||
public async compress(data: Buffer, level?: TCompressionLevel): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const options = level !== undefined ? { level } : undefined;
|
||||
plugins.fflate.gzip(data, options as plugins.fflate.AsyncGzipOptions, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(Buffer.from(result));
|
||||
});
|
||||
});
|
||||
// Use sync version wrapped in Promise for cross-runtime compatibility
|
||||
return this.compressSync(data, level);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress data asynchronously
|
||||
* Note: Uses sync version for Deno compatibility (fflate async uses Web Workers
|
||||
* which have issues in Deno)
|
||||
*/
|
||||
public async decompress(data: Buffer): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
plugins.fflate.gunzip(data, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(Buffer.from(result));
|
||||
});
|
||||
});
|
||||
// Use sync version wrapped in Promise for cross-runtime compatibility
|
||||
return this.decompressSync(data);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import type {
|
||||
IArchiveCreationOptions,
|
||||
IArchiveEntry,
|
||||
IArchiveExtractionOptions,
|
||||
IArchiveEntryInfo,
|
||||
IArchiveInfo,
|
||||
TArchiveFormat,
|
||||
TCompressionLevel,
|
||||
TEntryFilter,
|
||||
} from './interfaces.js';
|
||||
|
||||
import { Bzip2Tools } from './classes.bzip2tools.js';
|
||||
@@ -16,166 +15,48 @@ import { ZipTools } from './classes.ziptools.js';
|
||||
import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js';
|
||||
|
||||
/**
|
||||
* Main class for archive manipulation
|
||||
* Pending directory entry for async resolution
|
||||
*/
|
||||
interface IPendingDirectory {
|
||||
sourcePath: string;
|
||||
archiveBase?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main class for archive manipulation with fluent API
|
||||
* Supports TAR, ZIP, GZIP, and BZIP2 formats
|
||||
*
|
||||
* @example Extraction from URL
|
||||
* ```typescript
|
||||
* await SmartArchive.create()
|
||||
* .url('https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz')
|
||||
* .stripComponents(1)
|
||||
* .extract('./node_modules/lodash');
|
||||
* ```
|
||||
*
|
||||
* @example Creation with thenable
|
||||
* ```typescript
|
||||
* const archive = await SmartArchive.create()
|
||||
* .format('tar.gz')
|
||||
* .compression(9)
|
||||
* .entry('config.json', JSON.stringify(config))
|
||||
* .directory('./src');
|
||||
* ```
|
||||
*/
|
||||
export class SmartArchive {
|
||||
// ============================================
|
||||
// STATIC FACTORY METHODS - EXTRACTION
|
||||
// STATIC ENTRY POINT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Create SmartArchive from a URL
|
||||
* Create a new SmartArchive instance for fluent configuration
|
||||
*/
|
||||
public static async fromUrl(urlArg: string): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.sourceUrl = urlArg;
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SmartArchive from a local file path
|
||||
*/
|
||||
public static async fromFile(filePathArg: string): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.sourceFilePath = filePathArg;
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SmartArchive from a readable stream
|
||||
*/
|
||||
public static async fromStream(
|
||||
streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform
|
||||
): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.sourceStream = streamArg;
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SmartArchive from an in-memory buffer
|
||||
*/
|
||||
public static async fromBuffer(buffer: Buffer): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.sourceStream = plugins.stream.Readable.from(buffer);
|
||||
return smartArchiveInstance;
|
||||
public static create(): SmartArchive {
|
||||
return new SmartArchive();
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// STATIC FACTORY METHODS - CREATION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Create a new archive from a directory
|
||||
*/
|
||||
public static async fromDirectory(
|
||||
directoryPath: string,
|
||||
options: IArchiveCreationOptions
|
||||
): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.creationOptions = options;
|
||||
|
||||
const tarTools = new TarTools();
|
||||
|
||||
if (options.format === 'tar' || options.format === 'tar.gz' || options.format === 'tgz') {
|
||||
if (options.format === 'tar') {
|
||||
const pack = await tarTools.packDirectory(directoryPath);
|
||||
pack.finalize();
|
||||
smartArchiveInstance.archiveBuffer = await SmartArchive.streamToBuffer(pack);
|
||||
} else {
|
||||
smartArchiveInstance.archiveBuffer = await tarTools.packDirectoryToTarGz(
|
||||
directoryPath,
|
||||
options.compressionLevel
|
||||
);
|
||||
}
|
||||
} else if (options.format === 'zip') {
|
||||
const zipTools = new ZipTools();
|
||||
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
|
||||
const entries: IArchiveEntry[] = [];
|
||||
|
||||
for (const filePath of fileTree) {
|
||||
const absolutePath = plugins.path.join(directoryPath, filePath);
|
||||
const content = await plugins.fsPromises.readFile(absolutePath);
|
||||
entries.push({
|
||||
archivePath: filePath,
|
||||
content,
|
||||
});
|
||||
}
|
||||
|
||||
smartArchiveInstance.archiveBuffer = await zipTools.createZip(entries, options.compressionLevel);
|
||||
} else {
|
||||
throw new Error(`Unsupported format for directory packing: ${options.format}`);
|
||||
}
|
||||
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new archive from an array of entries
|
||||
*/
|
||||
public static async fromFiles(
|
||||
files: IArchiveEntry[],
|
||||
options: IArchiveCreationOptions
|
||||
): Promise<SmartArchive> {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.creationOptions = options;
|
||||
|
||||
if (options.format === 'tar' || options.format === 'tar.gz' || options.format === 'tgz') {
|
||||
const tarTools = new TarTools();
|
||||
if (options.format === 'tar') {
|
||||
smartArchiveInstance.archiveBuffer = await tarTools.packFiles(files);
|
||||
} else {
|
||||
smartArchiveInstance.archiveBuffer = await tarTools.packFilesToTarGz(files, options.compressionLevel);
|
||||
}
|
||||
} else if (options.format === 'zip') {
|
||||
const zipTools = new ZipTools();
|
||||
smartArchiveInstance.archiveBuffer = await zipTools.createZip(files, options.compressionLevel);
|
||||
} else if (options.format === 'gz') {
|
||||
if (files.length !== 1) {
|
||||
throw new Error('GZIP format only supports a single file');
|
||||
}
|
||||
const gzipTools = new GzipTools();
|
||||
let content: Buffer;
|
||||
if (typeof files[0].content === 'string') {
|
||||
content = Buffer.from(files[0].content);
|
||||
} else if (Buffer.isBuffer(files[0].content)) {
|
||||
content = files[0].content;
|
||||
} else {
|
||||
throw new Error('GZIP format requires string or Buffer content');
|
||||
}
|
||||
smartArchiveInstance.archiveBuffer = await gzipTools.compress(content, options.compressionLevel);
|
||||
} else {
|
||||
throw new Error(`Unsupported format: ${options.format}`);
|
||||
}
|
||||
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start building an archive incrementally using a builder pattern
|
||||
*/
|
||||
public static create(options: IArchiveCreationOptions): SmartArchive {
|
||||
const smartArchiveInstance = new SmartArchive();
|
||||
smartArchiveInstance.creationOptions = options;
|
||||
smartArchiveInstance.pendingEntries = [];
|
||||
return smartArchiveInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to convert a stream to buffer
|
||||
*/
|
||||
private static async streamToBuffer(stream: plugins.stream.Readable): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// INSTANCE PROPERTIES
|
||||
// TOOLS (public for internal use)
|
||||
// ============================================
|
||||
|
||||
public tarTools = new TarTools();
|
||||
@@ -184,129 +65,235 @@ export class SmartArchive {
|
||||
public bzip2Tools = new Bzip2Tools(this);
|
||||
public archiveAnalyzer = new ArchiveAnalyzer(this);
|
||||
|
||||
public sourceUrl?: string;
|
||||
public sourceFilePath?: string;
|
||||
public sourceStream?: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
|
||||
// ============================================
|
||||
// SOURCE STATE (extraction mode)
|
||||
// ============================================
|
||||
|
||||
private sourceUrl?: string;
|
||||
private sourceFilePath?: string;
|
||||
private sourceStream?: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform;
|
||||
|
||||
// ============================================
|
||||
// CREATION STATE
|
||||
// ============================================
|
||||
|
||||
private archiveBuffer?: Buffer;
|
||||
private creationOptions?: IArchiveCreationOptions;
|
||||
private pendingEntries?: IArchiveEntry[];
|
||||
private creationFormat?: TArchiveFormat;
|
||||
private _compressionLevel: TCompressionLevel = 6;
|
||||
private pendingEntries: IArchiveEntry[] = [];
|
||||
private pendingDirectories: IPendingDirectory[] = [];
|
||||
|
||||
// ============================================
|
||||
// FLUENT STATE
|
||||
// ============================================
|
||||
|
||||
private _mode: 'extract' | 'create' | null = null;
|
||||
private _filters: TEntryFilter[] = [];
|
||||
private _excludePatterns: RegExp[] = [];
|
||||
private _includePatterns: RegExp[] = [];
|
||||
private _stripComponents: number = 0;
|
||||
private _overwrite: boolean = false;
|
||||
private _fileName?: string;
|
||||
|
||||
constructor() {}
|
||||
|
||||
// ============================================
|
||||
// BUILDER METHODS (for incremental creation)
|
||||
// SOURCE METHODS (set extraction mode)
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Add a file to the archive (builder pattern)
|
||||
* Load archive from URL
|
||||
*/
|
||||
public addFile(archivePath: string, content: string | Buffer): this {
|
||||
if (!this.pendingEntries) {
|
||||
throw new Error('addFile can only be called on archives created with SmartArchive.create()');
|
||||
public url(urlArg: string): this {
|
||||
this.ensureNotInCreateMode('url');
|
||||
this._mode = 'extract';
|
||||
this.sourceUrl = urlArg;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load archive from file path
|
||||
*/
|
||||
public file(pathArg: string): this {
|
||||
this.ensureNotInCreateMode('file');
|
||||
this._mode = 'extract';
|
||||
this.sourceFilePath = pathArg;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load archive from readable stream
|
||||
*/
|
||||
public stream(streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform): this {
|
||||
this.ensureNotInCreateMode('stream');
|
||||
this._mode = 'extract';
|
||||
this.sourceStream = streamArg;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load archive from buffer
|
||||
*/
|
||||
public buffer(bufferArg: Buffer): this {
|
||||
this.ensureNotInCreateMode('buffer');
|
||||
this._mode = 'extract';
|
||||
this.sourceStream = plugins.stream.Readable.from(bufferArg);
|
||||
return this;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FORMAT METHODS (set creation mode)
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Set output format for archive creation
|
||||
*/
|
||||
public format(fmt: TArchiveFormat): this {
|
||||
this.ensureNotInExtractMode('format');
|
||||
this._mode = 'create';
|
||||
this.creationFormat = fmt;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set compression level (0-9)
|
||||
*/
|
||||
public compression(level: TCompressionLevel): this {
|
||||
this._compressionLevel = level;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CONTENT METHODS (creation mode)
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Add a single file entry to the archive
|
||||
*/
|
||||
public entry(archivePath: string, content: string | Buffer): this {
|
||||
this.ensureNotInExtractMode('entry');
|
||||
if (!this._mode) this._mode = 'create';
|
||||
this.pendingEntries.push({ archivePath, content });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a SmartFile to the archive (builder pattern)
|
||||
* Add multiple entries to the archive
|
||||
*/
|
||||
public addSmartFile(file: plugins.smartfile.SmartFile, archivePath?: string): this {
|
||||
if (!this.pendingEntries) {
|
||||
throw new Error('addSmartFile can only be called on archives created with SmartArchive.create()');
|
||||
public entries(entriesArg: Array<{ archivePath: string; content: string | Buffer }>): this {
|
||||
this.ensureNotInExtractMode('entries');
|
||||
if (!this._mode) this._mode = 'create';
|
||||
for (const e of entriesArg) {
|
||||
this.pendingEntries.push({ archivePath: e.archivePath, content: e.content });
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add an entire directory to the archive (queued, resolved at build time)
|
||||
*/
|
||||
public directory(sourcePath: string, archiveBase?: string): this {
|
||||
this.ensureNotInExtractMode('directory');
|
||||
if (!this._mode) this._mode = 'create';
|
||||
this.pendingDirectories.push({ sourcePath, archiveBase });
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a SmartFile to the archive
|
||||
*/
|
||||
public addSmartFile(fileArg: plugins.smartfile.SmartFile, archivePath?: string): this {
|
||||
this.ensureNotInExtractMode('addSmartFile');
|
||||
if (!this._mode) this._mode = 'create';
|
||||
this.pendingEntries.push({
|
||||
archivePath: archivePath || file.relative,
|
||||
content: file,
|
||||
archivePath: archivePath || fileArg.relative,
|
||||
content: fileArg,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a StreamFile to the archive (builder pattern)
|
||||
* Add a StreamFile to the archive
|
||||
*/
|
||||
public addStreamFile(file: plugins.smartfile.StreamFile, archivePath?: string): this {
|
||||
if (!this.pendingEntries) {
|
||||
throw new Error('addStreamFile can only be called on archives created with SmartArchive.create()');
|
||||
}
|
||||
public addStreamFile(fileArg: plugins.smartfile.StreamFile, archivePath?: string): this {
|
||||
this.ensureNotInExtractMode('addStreamFile');
|
||||
if (!this._mode) this._mode = 'create';
|
||||
this.pendingEntries.push({
|
||||
archivePath: archivePath || file.relativeFilePath,
|
||||
content: file,
|
||||
archivePath: archivePath || fileArg.relativeFilePath,
|
||||
content: fileArg,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILTER METHODS (both modes)
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Build the archive from pending entries
|
||||
* Filter entries by predicate function
|
||||
*/
|
||||
public async build(): Promise<SmartArchive> {
|
||||
if (!this.pendingEntries || !this.creationOptions) {
|
||||
throw new Error('build can only be called on archives created with SmartArchive.create()');
|
||||
public filter(predicate: TEntryFilter): this {
|
||||
this._filters.push(predicate);
|
||||
return this;
|
||||
}
|
||||
|
||||
const built = await SmartArchive.fromFiles(this.pendingEntries, this.creationOptions);
|
||||
this.archiveBuffer = built.archiveBuffer;
|
||||
this.pendingEntries = undefined;
|
||||
/**
|
||||
* Include only entries matching the pattern
|
||||
*/
|
||||
public include(pattern: string | RegExp): this {
|
||||
const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
|
||||
this._includePatterns.push(regex);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exclude entries matching the pattern
|
||||
*/
|
||||
public exclude(pattern: string | RegExp): this {
|
||||
const regex = typeof pattern === 'string' ? new RegExp(pattern) : pattern;
|
||||
this._excludePatterns.push(regex);
|
||||
return this;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXTRACTION METHODS
|
||||
// EXTRACTION OPTIONS
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Get the original archive stream
|
||||
* Strip N leading path components from extracted files
|
||||
*/
|
||||
public async toStream(): Promise<plugins.stream.Readable> {
|
||||
if (this.archiveBuffer) {
|
||||
return plugins.stream.Readable.from(this.archiveBuffer);
|
||||
}
|
||||
if (this.sourceStream) {
|
||||
return this.sourceStream;
|
||||
}
|
||||
if (this.sourceUrl) {
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(this.sourceUrl)
|
||||
.get();
|
||||
const webStream = response.stream();
|
||||
return plugins.stream.Readable.fromWeb(webStream as any);
|
||||
}
|
||||
if (this.sourceFilePath) {
|
||||
return plugins.fs.createReadStream(this.sourceFilePath);
|
||||
}
|
||||
throw new Error('No archive source configured');
|
||||
public stripComponents(n: number): this {
|
||||
this._stripComponents = n;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get archive as a Buffer
|
||||
* Overwrite existing files during extraction
|
||||
*/
|
||||
public async toBuffer(): Promise<Buffer> {
|
||||
if (this.archiveBuffer) {
|
||||
return this.archiveBuffer;
|
||||
}
|
||||
const stream = await this.toStream();
|
||||
return SmartArchive.streamToBuffer(stream);
|
||||
public overwrite(value: boolean = true): this {
|
||||
this._overwrite = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write archive to a file
|
||||
* Set output filename for single-file archives (gz, bz2)
|
||||
*/
|
||||
public async toFile(filePath: string): Promise<void> {
|
||||
const buffer = await this.toBuffer();
|
||||
await plugins.fsPromises.mkdir(plugins.path.dirname(filePath), { recursive: true });
|
||||
await plugins.fsPromises.writeFile(filePath, buffer);
|
||||
public fileName(name: string): this {
|
||||
this._fileName = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// TERMINAL METHODS - EXTRACTION
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Extract archive to filesystem
|
||||
* Extract archive to filesystem directory
|
||||
*/
|
||||
public async extractToDirectory(
|
||||
targetDir: string,
|
||||
options?: Partial<IArchiveExtractionOptions>
|
||||
): Promise<void> {
|
||||
public async extract(targetDir: string): Promise<void> {
|
||||
this.ensureExtractionSource();
|
||||
const done = plugins.smartpromise.defer<void>();
|
||||
const streamFileStream = await this.extractToStream();
|
||||
const streamFileStream = await this.toStreamFiles();
|
||||
|
||||
streamFileStream.pipe(
|
||||
new plugins.smartstream.SmartDuplex({
|
||||
@@ -314,27 +301,28 @@ export class SmartArchive {
|
||||
writeFunction: async (streamFileArg: plugins.smartfile.StreamFile) => {
|
||||
const innerDone = plugins.smartpromise.defer<void>();
|
||||
const streamFile = streamFileArg;
|
||||
let relativePath = streamFile.relativeFilePath || options?.fileName || 'extracted_file';
|
||||
let relativePath = streamFile.relativeFilePath || this._fileName || 'extracted_file';
|
||||
|
||||
// Apply stripComponents if specified
|
||||
if (options?.stripComponents && options.stripComponents > 0) {
|
||||
// Apply stripComponents
|
||||
if (this._stripComponents > 0) {
|
||||
const parts = relativePath.split('/');
|
||||
relativePath = parts.slice(options.stripComponents).join('/');
|
||||
relativePath = parts.slice(this._stripComponents).join('/');
|
||||
if (!relativePath) {
|
||||
innerDone.resolve();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filter if specified
|
||||
if (options?.filter) {
|
||||
// Apply filter
|
||||
const filterFn = this.buildFilterFunction();
|
||||
if (filterFn) {
|
||||
const entryInfo: IArchiveEntryInfo = {
|
||||
path: relativePath,
|
||||
size: 0,
|
||||
isDirectory: false,
|
||||
isFile: true,
|
||||
};
|
||||
if (!options.filter(entryInfo)) {
|
||||
if (!filterFn(entryInfo)) {
|
||||
innerDone.resolve();
|
||||
return;
|
||||
}
|
||||
@@ -363,7 +351,9 @@ export class SmartArchive {
|
||||
/**
|
||||
* Extract archive to a stream of StreamFile objects
|
||||
*/
|
||||
public async extractToStream(): Promise<plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>> {
|
||||
public async toStreamFiles(): Promise<plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>> {
|
||||
this.ensureExtractionSource();
|
||||
|
||||
const streamFileIntake = new plugins.smartstream.StreamIntake<plugins.smartfile.StreamFile>({
|
||||
objectMode: true,
|
||||
});
|
||||
@@ -377,7 +367,7 @@ export class SmartArchive {
|
||||
}
|
||||
};
|
||||
|
||||
const archiveStream = await this.toStream();
|
||||
const archiveStream = await this.getSourceStream();
|
||||
const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream();
|
||||
|
||||
const createUnpackStream = () =>
|
||||
@@ -447,20 +437,43 @@ export class SmartArchive {
|
||||
/**
|
||||
* Extract archive to an array of SmartFile objects (in-memory)
|
||||
*/
|
||||
public async extractToSmartFiles(): Promise<plugins.smartfile.SmartFile[]> {
|
||||
const streamFiles = await this.extractToStream();
|
||||
public async toSmartFiles(): Promise<plugins.smartfile.SmartFile[]> {
|
||||
this.ensureExtractionSource();
|
||||
const streamFiles = await this.toStreamFiles();
|
||||
const smartFiles: plugins.smartfile.SmartFile[] = [];
|
||||
const filterFn = this.buildFilterFunction();
|
||||
const pendingConversions: Promise<void>[] = [];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
streamFiles.on('data', async (streamFile: plugins.smartfile.StreamFile) => {
|
||||
streamFiles.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||
// Track all async conversions to ensure they complete before resolving
|
||||
const conversion = (async () => {
|
||||
try {
|
||||
const smartFile = await streamFile.toSmartFile();
|
||||
|
||||
// Apply filter if configured
|
||||
if (filterFn) {
|
||||
const passes = filterFn({
|
||||
path: smartFile.relative,
|
||||
size: smartFile.contents.length,
|
||||
isDirectory: false,
|
||||
isFile: true,
|
||||
});
|
||||
if (!passes) return;
|
||||
}
|
||||
|
||||
smartFiles.push(smartFile);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
})();
|
||||
pendingConversions.push(conversion);
|
||||
});
|
||||
streamFiles.on('end', async () => {
|
||||
// Wait for all conversions to complete before resolving
|
||||
await Promise.all(pendingConversions);
|
||||
resolve(smartFiles);
|
||||
});
|
||||
streamFiles.on('end', () => resolve(smartFiles));
|
||||
streamFiles.on('error', reject);
|
||||
});
|
||||
}
|
||||
@@ -469,7 +482,8 @@ export class SmartArchive {
|
||||
* Extract a single file from the archive by path
|
||||
*/
|
||||
public async extractFile(filePath: string): Promise<plugins.smartfile.SmartFile | null> {
|
||||
const streamFiles = await this.extractToStream();
|
||||
this.ensureExtractionSource();
|
||||
const streamFiles = await this.toStreamFiles();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let found = false;
|
||||
@@ -497,14 +511,114 @@ export class SmartArchive {
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ANALYSIS METHODS
|
||||
// TERMINAL METHODS - OUTPUT
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Build and finalize the archive, returning this instance
|
||||
*/
|
||||
public async build(): Promise<SmartArchive> {
|
||||
await this.doBuild();
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal build implementation (avoids thenable recursion)
|
||||
*/
|
||||
private async doBuild(): Promise<void> {
|
||||
if (this._mode === 'extract') {
|
||||
// For extraction mode, nothing to build
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.archiveBuffer) {
|
||||
// Already built
|
||||
return;
|
||||
}
|
||||
|
||||
// For creation mode, build the archive buffer
|
||||
this.ensureCreationFormat();
|
||||
await this.resolveDirectories();
|
||||
|
||||
const entries = this.getFilteredEntries();
|
||||
|
||||
if (this.creationFormat === 'tar' || this.creationFormat === 'tar.gz' || this.creationFormat === 'tgz') {
|
||||
if (this.creationFormat === 'tar') {
|
||||
this.archiveBuffer = await this.tarTools.packFiles(entries);
|
||||
} else {
|
||||
this.archiveBuffer = await this.tarTools.packFilesToTarGz(entries, this._compressionLevel);
|
||||
}
|
||||
} else if (this.creationFormat === 'zip') {
|
||||
this.archiveBuffer = await this.zipTools.createZip(entries, this._compressionLevel);
|
||||
} else if (this.creationFormat === 'gz') {
|
||||
if (entries.length !== 1) {
|
||||
throw new Error('GZIP format only supports a single file');
|
||||
}
|
||||
let content: Buffer;
|
||||
if (typeof entries[0].content === 'string') {
|
||||
content = Buffer.from(entries[0].content);
|
||||
} else if (Buffer.isBuffer(entries[0].content)) {
|
||||
content = entries[0].content;
|
||||
} else {
|
||||
throw new Error('GZIP format requires string or Buffer content');
|
||||
}
|
||||
this.archiveBuffer = await this.gzipTools.compress(content, this._compressionLevel);
|
||||
} else {
|
||||
throw new Error(`Unsupported format: ${this.creationFormat}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build archive and return as Buffer
|
||||
*/
|
||||
public async toBuffer(): Promise<Buffer> {
|
||||
if (this._mode === 'create' && !this.archiveBuffer) {
|
||||
await this.doBuild();
|
||||
}
|
||||
|
||||
if (this.archiveBuffer) {
|
||||
return this.archiveBuffer;
|
||||
}
|
||||
|
||||
// For extraction mode, get the source as buffer
|
||||
const stream = await this.getSourceStream();
|
||||
return this.streamToBuffer(stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build archive and write to file
|
||||
*/
|
||||
public async toFile(filePath: string): Promise<void> {
|
||||
const buffer = await this.toBuffer();
|
||||
await plugins.fsPromises.mkdir(plugins.path.dirname(filePath), { recursive: true });
|
||||
await plugins.fsPromises.writeFile(filePath, buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get archive as a readable stream
|
||||
*/
|
||||
public async toStream(): Promise<plugins.stream.Readable> {
|
||||
if (this._mode === 'create' && !this.archiveBuffer) {
|
||||
await this.doBuild();
|
||||
}
|
||||
|
||||
if (this.archiveBuffer) {
|
||||
return plugins.stream.Readable.from(this.archiveBuffer);
|
||||
}
|
||||
|
||||
return this.getSourceStream();
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// TERMINAL METHODS - ANALYSIS
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Analyze the archive and return metadata
|
||||
*/
|
||||
public async analyze(): Promise<IArchiveInfo> {
|
||||
const stream = await this.toStream();
|
||||
this.ensureExtractionSource();
|
||||
const stream = await this.getSourceStream();
|
||||
const firstChunk = await this.readFirstChunk(stream);
|
||||
const fileType = await plugins.fileType.fileTypeFromBuffer(firstChunk);
|
||||
|
||||
@@ -544,11 +658,12 @@ export class SmartArchive {
|
||||
}
|
||||
|
||||
/**
|
||||
* List all entries in the archive without extracting
|
||||
* List all entries in the archive
|
||||
*/
|
||||
public async listEntries(): Promise<IArchiveEntryInfo[]> {
|
||||
public async list(): Promise<IArchiveEntryInfo[]> {
|
||||
this.ensureExtractionSource();
|
||||
const entries: IArchiveEntryInfo[] = [];
|
||||
const streamFiles = await this.extractToStream();
|
||||
const streamFiles = await this.toStreamFiles();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
streamFiles.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||
@@ -568,12 +683,171 @@ export class SmartArchive {
|
||||
* Check if a specific file exists in the archive
|
||||
*/
|
||||
public async hasFile(filePath: string): Promise<boolean> {
|
||||
const entries = await this.listEntries();
|
||||
this.ensureExtractionSource();
|
||||
const entries = await this.list();
|
||||
return entries.some((e) => e.path === filePath || e.path.endsWith(filePath));
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// PRIVATE HELPERS
|
||||
// ============================================
|
||||
|
||||
/**
|
||||
* Helper to read first chunk from stream
|
||||
* Ensure we're not in create mode when calling extraction methods
|
||||
*/
|
||||
private ensureNotInCreateMode(methodName: string): void {
|
||||
if (this._mode === 'create') {
|
||||
throw new Error(
|
||||
`Cannot call .${methodName}() in creation mode. ` +
|
||||
`Use extraction methods (.url(), .file(), .stream(), .buffer()) for extraction mode.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure we're not in extract mode when calling creation methods
|
||||
*/
|
||||
private ensureNotInExtractMode(methodName: string): void {
|
||||
if (this._mode === 'extract') {
|
||||
throw new Error(
|
||||
`Cannot call .${methodName}() in extraction mode. ` +
|
||||
`Use .format() for creation mode.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure an extraction source is configured
|
||||
*/
|
||||
private ensureExtractionSource(): void {
|
||||
if (!this.sourceUrl && !this.sourceFilePath && !this.sourceStream && !this.archiveBuffer) {
|
||||
throw new Error(
|
||||
'No source configured. Call .url(), .file(), .stream(), or .buffer() first.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a format is configured for creation
|
||||
*/
|
||||
private ensureCreationFormat(): void {
|
||||
if (!this.creationFormat) {
|
||||
throw new Error('No format specified. Call .format() before creating archive.');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the source stream
|
||||
*/
|
||||
private async getSourceStream(): Promise<plugins.stream.Readable> {
|
||||
if (this.archiveBuffer) {
|
||||
return plugins.stream.Readable.from(this.archiveBuffer);
|
||||
}
|
||||
if (this.sourceStream) {
|
||||
return this.sourceStream;
|
||||
}
|
||||
if (this.sourceUrl) {
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url(this.sourceUrl)
|
||||
.get();
|
||||
const webStream = response.stream();
|
||||
return plugins.stream.Readable.fromWeb(webStream as any);
|
||||
}
|
||||
if (this.sourceFilePath) {
|
||||
return plugins.fs.createReadStream(this.sourceFilePath);
|
||||
}
|
||||
throw new Error('No archive source configured');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a combined filter function from all configured filters
|
||||
*/
|
||||
private buildFilterFunction(): TEntryFilter | undefined {
|
||||
const hasFilters =
|
||||
this._filters.length > 0 ||
|
||||
this._includePatterns.length > 0 ||
|
||||
this._excludePatterns.length > 0;
|
||||
|
||||
if (!hasFilters) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return (entry: IArchiveEntryInfo) => {
|
||||
// Check include patterns (if any specified, at least one must match)
|
||||
if (this._includePatterns.length > 0) {
|
||||
const included = this._includePatterns.some((p) => p.test(entry.path));
|
||||
if (!included) return false;
|
||||
}
|
||||
|
||||
// Check exclude patterns (none must match)
|
||||
for (const pattern of this._excludePatterns) {
|
||||
if (pattern.test(entry.path)) return false;
|
||||
}
|
||||
|
||||
// Check custom filters (all must pass)
|
||||
for (const filter of this._filters) {
|
||||
if (!filter(entry)) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve pending directories to entries
|
||||
*/
|
||||
private async resolveDirectories(): Promise<void> {
|
||||
for (const dir of this.pendingDirectories) {
|
||||
const files = await plugins.listFileTree(dir.sourcePath, '**/*');
|
||||
for (const filePath of files) {
|
||||
const archivePath = dir.archiveBase
|
||||
? plugins.path.join(dir.archiveBase, filePath)
|
||||
: filePath;
|
||||
const absolutePath = plugins.path.join(dir.sourcePath, filePath);
|
||||
const content = await plugins.fsPromises.readFile(absolutePath);
|
||||
this.pendingEntries.push({
|
||||
archivePath,
|
||||
content,
|
||||
});
|
||||
}
|
||||
}
|
||||
this.pendingDirectories = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get entries filtered by include/exclude patterns
|
||||
*/
|
||||
private getFilteredEntries(): IArchiveEntry[] {
|
||||
const filterFn = this.buildFilterFunction();
|
||||
if (!filterFn) {
|
||||
return this.pendingEntries;
|
||||
}
|
||||
|
||||
return this.pendingEntries.filter((entry) =>
|
||||
filterFn({
|
||||
path: entry.archivePath,
|
||||
size: 0,
|
||||
isDirectory: false,
|
||||
isFile: true,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a stream to buffer
|
||||
*/
|
||||
private async streamToBuffer(stream: plugins.stream.Readable): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read first chunk from stream
|
||||
*/
|
||||
private async readFirstChunk(stream: plugins.stream.Readable): Promise<Buffer> {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
@@ -100,17 +100,14 @@ export class ZipCompressionStream extends plugins.stream.Duplex {
|
||||
filesObj[name] = options ? [data, options] : data;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
plugins.fflate.zip(filesObj, (err, result) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||
try {
|
||||
const result = plugins.fflate.zipSync(filesObj);
|
||||
this.push(Buffer.from(result));
|
||||
this.push(null);
|
||||
resolve();
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_read(): void {
|
||||
@@ -179,31 +176,21 @@ export class ZipTools {
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
plugins.fflate.zip(filesObj, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(Buffer.from(result));
|
||||
});
|
||||
});
|
||||
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||
const result = plugins.fflate.zipSync(filesObj);
|
||||
return Buffer.from(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a ZIP buffer to an array of entries
|
||||
*/
|
||||
public async extractZip(data: Buffer): Promise<Array<{ path: string; content: Buffer }>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
plugins.fflate.unzip(data, (err, result) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
}
|
||||
|
||||
// Use sync version for Deno compatibility (fflate async uses Web Workers)
|
||||
const result = plugins.fflate.unzipSync(data);
|
||||
const entries: Array<{ path: string; content: Buffer }> = [];
|
||||
for (const [path, content] of Object.entries(result)) {
|
||||
entries.push({ path, content: Buffer.from(content) });
|
||||
}
|
||||
resolve(entries);
|
||||
});
|
||||
});
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,3 +129,8 @@ export interface IHuffmanGroup {
|
||||
minLen: number;
|
||||
maxLen: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry filter predicate for fluent API
|
||||
*/
|
||||
export type TEntryFilter = (entry: IArchiveEntryInfo) => boolean;
|
||||
|
||||
Reference in New Issue
Block a user