Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e609c023bc | |||
| 634c204a00 | |||
| 7ccd210c45 | |||
| fe90de56d6 | |||
| d9251fa1a5 | |||
| ec58b9cdc5 |
27
changelog.md
27
changelog.md
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-11-25 - 4.2.4 - fix(plugins)
|
||||
Migrate filesystem usage to Node fs/fsPromises and upgrade smartfile to v13; add listFileTree helper and update tests
|
||||
|
||||
- Bumped dependency @push.rocks/smartfile to ^13.0.0 and removed unused dependency `through`
|
||||
- Replaced usages of smartfile.fs and smartfile.fsStream with Node native fs and fs/promises (createReadStream/createWriteStream, mkdir({recursive:true}), stat, readFile)
|
||||
- Added plugins.listFileTree helper (recursive directory lister) and used it in TarTools.packDirectory and tests
|
||||
- Updated SmartArchive.exportToFs to use plugins.fs and plugins.fsPromises for directory creation and file writes
|
||||
- Updated TarTools to use plugins.fs.createReadStream and plugins.fsPromises.stat when packing directories
|
||||
- Converted/updated tests to a Node/Deno-friendly test file (test.node+deno.ts) and switched test helpers to use fsPromises
|
||||
- Added readme.hints.md with migration notes for Smartfile v13 and architecture/dependency notes
|
||||
|
||||
## 2025-11-25 - 4.2.3 - fix(build)
|
||||
Upgrade dev tooling: bump @git.zone/tsbuild, @git.zone/tsrun and @git.zone/tstest versions
|
||||
|
||||
- Bump @git.zone/tsbuild from ^2.6.6 to ^3.1.0
|
||||
- Bump @git.zone/tsrun from ^1.3.3 to ^2.0.0
|
||||
- Bump @git.zone/tstest from ^2.3.4 to ^3.1.3
|
||||
|
||||
## 2025-08-18 - 4.2.2 - fix(smartarchive)
|
||||
Improve tar entry streaming handling and add in-memory gzip/tgz tests
|
||||
|
||||
- Fix tar entry handling: properly consume directory entries (resume stream) and wait for entry end before continuing to next header
|
||||
- Wrap tar file entries with a PassThrough so extracted StreamFile instances can be consumed while the tar extractor continues
|
||||
- Handle nested archives correctly by piping resultStream -> decompressionStream -> analyzer -> unpacker, avoiding premature end signals
|
||||
- Add and expand tests in test/test.gzip.ts: verify package.json and TS/license files after extraction, add in-memory gzip extraction test, and add real tgz-in-memory download+extraction test
|
||||
- Minor logging improvements for tar extraction flow
|
||||
|
||||
## 2025-08-18 - 4.2.1 - fix(gzip)
|
||||
Improve gzip streaming decompression, archive analysis and unpacking; add gzip tests
|
||||
|
||||
|
||||
13
package.json
13
package.json
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@push.rocks/smartarchive",
|
||||
"version": "4.2.1",
|
||||
"version": "4.2.4",
|
||||
"description": "A library for working with archive files, providing utilities for compressing and decompressing data.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
@@ -22,7 +22,7 @@
|
||||
"homepage": "https://code.foss.global/push.rocks/smartarchive#readme",
|
||||
"dependencies": {
|
||||
"@push.rocks/smartdelay": "^3.0.5",
|
||||
"@push.rocks/smartfile": "^11.2.7",
|
||||
"@push.rocks/smartfile": "^13.0.0",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartpromise": "^4.2.3",
|
||||
"@push.rocks/smartrequest": "^4.2.2",
|
||||
@@ -33,13 +33,12 @@
|
||||
"@types/tar-stream": "^3.1.4",
|
||||
"fflate": "^0.8.2",
|
||||
"file-type": "^21.0.0",
|
||||
"tar-stream": "^3.1.7",
|
||||
"through": "^2.3.8"
|
||||
"tar-stream": "^3.1.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^2.6.6",
|
||||
"@git.zone/tsrun": "^1.3.3",
|
||||
"@git.zone/tstest": "^2.3.4"
|
||||
"@git.zone/tsbuild": "^3.1.0",
|
||||
"@git.zone/tsrun": "^2.0.0",
|
||||
"@git.zone/tstest": "^3.1.3"
|
||||
},
|
||||
"private": false,
|
||||
"files": [
|
||||
|
||||
3586
pnpm-lock.yaml
generated
3586
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1 +1,38 @@
|
||||
|
||||
# Smartarchive Development Hints
|
||||
|
||||
## Dependency Upgrades (2025-01-25)
|
||||
|
||||
### Completed Upgrades
|
||||
- **@git.zone/tsbuild**: ^2.6.6 → ^3.1.0
|
||||
- **@git.zone/tsrun**: ^1.3.3 → ^2.0.0
|
||||
- **@git.zone/tstest**: ^2.3.4 → ^3.1.3
|
||||
- **@push.rocks/smartfile**: ^11.2.7 → ^13.0.0
|
||||
|
||||
### Migration Notes
|
||||
|
||||
#### Smartfile v13 Migration
|
||||
Smartfile v13 removed filesystem operations (`fs`, `memory`, `fsStream` namespaces). These were replaced with Node.js native `fs` and `fs/promises`:
|
||||
|
||||
**Replacements made:**
|
||||
- `smartfile.fs.ensureDir(path)` → `fsPromises.mkdir(path, { recursive: true })`
|
||||
- `smartfile.fs.stat(path)` → `fsPromises.stat(path)`
|
||||
- `smartfile.fs.toReadStream(path)` → `fs.createReadStream(path)`
|
||||
- `smartfile.fs.toStringSync(path)` → `fsPromises.readFile(path, 'utf8')`
|
||||
- `smartfile.fs.listFileTree(dir, pattern)` → custom `listFileTree()` helper
|
||||
- `smartfile.fsStream.createReadStream(path)` → `fs.createReadStream(path)`
|
||||
- `smartfile.fsStream.createWriteStream(path)` → `fs.createWriteStream(path)`
|
||||
- `smartfile.memory.toFs(content, path)` → `fsPromises.writeFile(path, content)`
|
||||
|
||||
**Still using from smartfile v13:**
|
||||
- `SmartFile` class (in-memory file representation)
|
||||
- `StreamFile` class (streaming file handling)
|
||||
|
||||
### Removed Dependencies
|
||||
- `through@2.3.8` - was unused in the codebase
|
||||
|
||||
## Architecture Notes
|
||||
|
||||
- Uses `fflate` for ZIP/GZIP compression (pure JS, works in browser)
|
||||
- Uses `tar-stream` for TAR archive handling
|
||||
- Uses `file-type` for MIME type detection
|
||||
- Custom BZIP2 implementation in `ts/bzip2/` directory
|
||||
|
||||
411
readme.md
411
readme.md
@@ -1,29 +1,32 @@
|
||||
# @push.rocks/smartarchive 📦
|
||||
|
||||
**Powerful archive manipulation for modern Node.js applications**
|
||||
Powerful archive manipulation for modern Node.js applications.
|
||||
|
||||
`@push.rocks/smartarchive` is a versatile library for handling archive files with a focus on developer experience. Work with **zip**, **tar**, **gzip**, and **bzip2** formats through a unified, streaming-optimized API.
|
||||
|
||||
## Issue Reporting and Security
|
||||
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
## Features 🚀
|
||||
|
||||
- 📁 **Multi-format support** - Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, and `.bz2` archives
|
||||
- 🌊 **Streaming-first architecture** - Process large archives without memory constraints
|
||||
- 🔄 **Unified API** - Consistent interface across different archive formats
|
||||
- 🎯 **Smart detection** - Automatically identifies archive types
|
||||
- ⚡ **High performance** - Optimized for speed with parallel processing where possible
|
||||
- 🔧 **Flexible I/O** - Work with files, URLs, and streams seamlessly
|
||||
- 📊 **Archive analysis** - Inspect contents without extraction
|
||||
- 🛠️ **Modern TypeScript** - Full type safety and excellent IDE support
|
||||
- 📁 **Multi-format support** – Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, and `.bz2` archives
|
||||
- 🌊 **Streaming-first architecture** – Process large archives without memory constraints
|
||||
- 🔄 **Unified API** – Consistent interface across different archive formats
|
||||
- 🎯 **Smart detection** – Automatically identifies archive types via magic bytes
|
||||
- ⚡ **High performance** – Built on `tar-stream` and `fflate` for speed
|
||||
- 🔧 **Flexible I/O** – Work with files, URLs, and streams seamlessly
|
||||
- 🛠️ **Modern TypeScript** – Full type safety and excellent IDE support
|
||||
|
||||
## Installation 📥
|
||||
|
||||
```bash
|
||||
# Using npm
|
||||
npm install @push.rocks/smartarchive
|
||||
|
||||
# Using pnpm (recommended)
|
||||
pnpm add @push.rocks/smartarchive
|
||||
|
||||
# Using npm
|
||||
npm install @push.rocks/smartarchive
|
||||
|
||||
# Using yarn
|
||||
yarn add @push.rocks/smartarchive
|
||||
```
|
||||
@@ -37,7 +40,7 @@ import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Extract a .tar.gz archive from a URL directly to the filesystem
|
||||
const archive = await SmartArchive.fromArchiveUrl(
|
||||
'https://github.com/some/repo/archive/main.tar.gz'
|
||||
'https://registry.npmjs.org/some-package/-/some-package-1.0.0.tgz'
|
||||
);
|
||||
await archive.exportToFs('./extracted');
|
||||
```
|
||||
@@ -52,10 +55,15 @@ const archive = await SmartArchive.fromArchiveFile('./large-archive.zip');
|
||||
const streamOfFiles = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
// Process each file in the archive
|
||||
streamOfFiles.on('data', (fileStream) => {
|
||||
console.log(`Processing ${fileStream.path}`);
|
||||
streamOfFiles.on('data', async (streamFile) => {
|
||||
console.log(`Processing ${streamFile.relativeFilePath}`);
|
||||
const readStream = await streamFile.createReadStream();
|
||||
// Handle individual file stream
|
||||
});
|
||||
|
||||
streamOfFiles.on('end', () => {
|
||||
console.log('Extraction complete');
|
||||
});
|
||||
```
|
||||
|
||||
## Core Concepts 💡
|
||||
@@ -64,17 +72,18 @@ streamOfFiles.on('data', (fileStream) => {
|
||||
|
||||
`SmartArchive` accepts archives from three sources:
|
||||
|
||||
1. **URL** - Download and process archives from the web
|
||||
2. **File** - Load archives from the local filesystem
|
||||
3. **Stream** - Process archives from any Node.js stream
|
||||
| Source | Method | Use Case |
|
||||
|--------|--------|----------|
|
||||
| **URL** | `SmartArchive.fromArchiveUrl(url)` | Download and process archives from the web |
|
||||
| **File** | `SmartArchive.fromArchiveFile(path)` | Load archives from the local filesystem |
|
||||
| **Stream** | `SmartArchive.fromArchiveStream(stream)` | Process archives from any Node.js stream |
|
||||
|
||||
### Export Destinations
|
||||
|
||||
Extract archives to multiple destinations:
|
||||
|
||||
1. **Filesystem** - Extract directly to a directory
|
||||
2. **Stream of files** - Process files individually as streams
|
||||
3. **Archive stream** - Re-stream as different format
|
||||
| Destination | Method | Use Case |
|
||||
|-------------|--------|----------|
|
||||
| **Filesystem** | `exportToFs(targetDir, fileName?)` | Extract directly to a directory |
|
||||
| **Stream of files** | `exportToStreamOfStreamFiles()` | Process files individually as `StreamFile` objects |
|
||||
|
||||
## Usage Examples 🔨
|
||||
|
||||
@@ -89,10 +98,11 @@ await zipArchive.exportToFs('./output');
|
||||
|
||||
// Stream ZIP contents for processing
|
||||
const fileStream = await zipArchive.exportToStreamOfStreamFiles();
|
||||
fileStream.on('data', (file) => {
|
||||
if (file.path.endsWith('.json')) {
|
||||
|
||||
fileStream.on('data', async (streamFile) => {
|
||||
if (streamFile.relativeFilePath.endsWith('.json')) {
|
||||
const readStream = await streamFile.createReadStream();
|
||||
// Process JSON files from the archive
|
||||
file.pipe(jsonProcessor);
|
||||
}
|
||||
});
|
||||
```
|
||||
@@ -106,10 +116,38 @@ import { SmartArchive, TarTools } from '@push.rocks/smartarchive';
|
||||
const tarGzArchive = await SmartArchive.fromArchiveFile('./archive.tar.gz');
|
||||
await tarGzArchive.exportToFs('./extracted');
|
||||
|
||||
// Create a TAR archive (using TarTools directly)
|
||||
// Create a TAR archive using TarTools directly
|
||||
const tarTools = new TarTools();
|
||||
const packStream = await tarTools.packDirectory('./source-directory');
|
||||
packStream.pipe(createWriteStream('./output.tar'));
|
||||
const pack = await tarTools.getPackStream();
|
||||
|
||||
// Add files to the pack
|
||||
await tarTools.addFileToPack(pack, {
|
||||
fileName: 'hello.txt',
|
||||
content: 'Hello, World!'
|
||||
});
|
||||
|
||||
await tarTools.addFileToPack(pack, {
|
||||
fileName: 'data.json',
|
||||
content: Buffer.from(JSON.stringify({ foo: 'bar' }))
|
||||
});
|
||||
|
||||
// Finalize and pipe to destination
|
||||
pack.finalize();
|
||||
pack.pipe(createWriteStream('./output.tar'));
|
||||
```
|
||||
|
||||
### Pack a directory into TAR
|
||||
|
||||
```typescript
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
import { createWriteStream } from 'fs';
|
||||
|
||||
const tarTools = new TarTools();
|
||||
|
||||
// Pack an entire directory
|
||||
const pack = await tarTools.packDirectory('./src');
|
||||
pack.finalize();
|
||||
pack.pipe(createWriteStream('./source.tar'));
|
||||
```
|
||||
|
||||
### Extracting from URLs
|
||||
@@ -117,47 +155,36 @@ packStream.pipe(createWriteStream('./output.tar'));
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Download and extract in one operation
|
||||
const remoteArchive = await SmartArchive.fromArchiveUrl(
|
||||
'https://example.com/data.tar.gz'
|
||||
// Download and extract npm packages
|
||||
const npmPackage = await SmartArchive.fromArchiveUrl(
|
||||
'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz'
|
||||
);
|
||||
await npmPackage.exportToFs('./node_modules/@push.rocks/smartfile');
|
||||
|
||||
// Extract to filesystem
|
||||
await remoteArchive.exportToFs('./local-dir');
|
||||
|
||||
// Or process as stream
|
||||
const stream = await remoteArchive.exportToStreamOfStreamFiles();
|
||||
```
|
||||
|
||||
### Analyzing archive contents
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Analyze without extracting
|
||||
const archive = await SmartArchive.fromArchiveFile('./archive.zip');
|
||||
const analyzer = archive.archiveAnalyzer;
|
||||
|
||||
// Use the analyzer to inspect contents
|
||||
// (exact implementation depends on analyzer methods)
|
||||
// Or process as stream for memory efficiency
|
||||
const stream = await npmPackage.exportToStreamOfStreamFiles();
|
||||
stream.on('data', async (file) => {
|
||||
console.log(`Extracted: ${file.relativeFilePath}`);
|
||||
});
|
||||
```
|
||||
|
||||
### Working with GZIP files
|
||||
|
||||
```typescript
|
||||
import { SmartArchive, GzipTools } from '@push.rocks/smartarchive';
|
||||
import { createReadStream, createWriteStream } from 'fs';
|
||||
|
||||
// Decompress a .gz file
|
||||
// Decompress a .gz file - provide filename since gzip doesn't store it
|
||||
const gzipArchive = await SmartArchive.fromArchiveFile('./data.json.gz');
|
||||
await gzipArchive.exportToFs('./decompressed', 'data.json');
|
||||
|
||||
// Use GzipTools directly for streaming
|
||||
// Use GzipTools directly for streaming decompression
|
||||
const gzipTools = new GzipTools();
|
||||
const decompressStream = gzipTools.getDecompressionStream();
|
||||
|
||||
createReadStream('./compressed.gz')
|
||||
.pipe(decompressStream)
|
||||
.pipe(createWriteStream('./decompressed'));
|
||||
.pipe(createWriteStream('./decompressed.txt'));
|
||||
```
|
||||
|
||||
### Working with BZIP2 files
|
||||
@@ -172,115 +199,175 @@ const bzipArchive = await SmartArchive.fromArchiveUrl(
|
||||
await bzipArchive.exportToFs('./extracted', 'data.txt');
|
||||
```
|
||||
|
||||
### Advanced streaming operations
|
||||
### In-memory processing (no filesystem)
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
// Chain operations with streams
|
||||
const archive = await SmartArchive.fromArchiveFile('./archive.tar.gz');
|
||||
const exportStream = await archive.exportToStreamOfStreamFiles();
|
||||
// Process archives entirely in memory
|
||||
const compressedBuffer = await fetchCompressedData();
|
||||
const memoryStream = Readable.from(compressedBuffer);
|
||||
|
||||
// Process each file in the archive
|
||||
await pipeline(
|
||||
exportStream,
|
||||
async function* (source) {
|
||||
for await (const file of source) {
|
||||
if (file.path.endsWith('.log')) {
|
||||
// Process log files
|
||||
yield processLogFile(file);
|
||||
}
|
||||
}
|
||||
},
|
||||
createWriteStream('./processed-logs.txt')
|
||||
);
|
||||
```
|
||||
const archive = await SmartArchive.fromArchiveStream(memoryStream);
|
||||
const streamFiles = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
### Creating archives (advanced)
|
||||
const extractedFiles: Array<{ name: string; content: Buffer }> = [];
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
streamFiles.on('data', async (streamFile) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await streamFile.createReadStream();
|
||||
|
||||
// Using SmartArchive to create an archive
|
||||
const archive = new SmartArchive();
|
||||
|
||||
// Add content to the archive
|
||||
archive.addedDirectories.push('./src');
|
||||
archive.addedFiles.push('./readme.md');
|
||||
archive.addedFiles.push('./package.json');
|
||||
|
||||
// Export as TAR.GZ
|
||||
const tarGzStream = await archive.exportToTarGzStream();
|
||||
tarGzStream.pipe(createWriteStream('./output.tar.gz'));
|
||||
```
|
||||
|
||||
### Extract and transform
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
import { Transform } from 'stream';
|
||||
|
||||
// Extract and transform files in one pipeline
|
||||
const archive = await SmartArchive.fromArchiveUrl(
|
||||
'https://example.com/source-code.tar.gz'
|
||||
);
|
||||
|
||||
const extractStream = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
// Transform TypeScript to JavaScript during extraction
|
||||
extractStream.on('data', (fileStream) => {
|
||||
if (fileStream.path.endsWith('.ts')) {
|
||||
fileStream
|
||||
.pipe(typescriptTranspiler())
|
||||
.pipe(createWriteStream(fileStream.path.replace('.ts', '.js')));
|
||||
} else {
|
||||
fileStream.pipe(createWriteStream(fileStream.path));
|
||||
for await (const chunk of readStream) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
|
||||
extractedFiles.push({
|
||||
name: streamFile.relativeFilePath,
|
||||
content: Buffer.concat(chunks)
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise((resolve) => streamFiles.on('end', resolve));
|
||||
console.log(`Extracted ${extractedFiles.length} files in memory`);
|
||||
```
|
||||
|
||||
### Nested archive handling (e.g., .tar.gz)
|
||||
|
||||
The library automatically handles nested compression. A `.tar.gz` file is:
|
||||
1. First decompressed from gzip
|
||||
2. Then unpacked from tar
|
||||
|
||||
This happens transparently:
|
||||
|
||||
```typescript
|
||||
import { SmartArchive } from '@push.rocks/smartarchive';
|
||||
|
||||
// Automatically handles gzip → tar extraction chain
|
||||
const tgzArchive = await SmartArchive.fromArchiveFile('./package.tar.gz');
|
||||
await tgzArchive.exportToFs('./extracted');
|
||||
```
|
||||
|
||||
## API Reference 📚
|
||||
|
||||
### SmartArchive Class
|
||||
|
||||
#### Static Methods
|
||||
The main entry point for archive operations.
|
||||
|
||||
- `SmartArchive.fromArchiveUrl(url: string)` - Create from URL
|
||||
- `SmartArchive.fromArchiveFile(path: string)` - Create from file
|
||||
- `SmartArchive.fromArchiveStream(stream: NodeJS.ReadableStream)` - Create from stream
|
||||
#### Static Factory Methods
|
||||
|
||||
```typescript
|
||||
// Create from URL - downloads and processes archive
|
||||
SmartArchive.fromArchiveUrl(url: string): Promise<SmartArchive>
|
||||
|
||||
// Create from local file path
|
||||
SmartArchive.fromArchiveFile(path: string): Promise<SmartArchive>
|
||||
|
||||
// Create from any Node.js readable stream
|
||||
SmartArchive.fromArchiveStream(stream: Readable | Duplex | Transform): Promise<SmartArchive>
|
||||
```
|
||||
|
||||
#### Instance Methods
|
||||
|
||||
- `exportToFs(targetDir: string, fileName?: string)` - Extract to filesystem
|
||||
- `exportToStreamOfStreamFiles()` - Get a stream of file streams
|
||||
- `exportToTarGzStream()` - Export as TAR.GZ stream
|
||||
- `getArchiveStream()` - Get the raw archive stream
|
||||
```typescript
|
||||
// Extract all files to a directory
|
||||
// fileName is optional - used for single-file archives (like .gz) that don't store filename
|
||||
exportToFs(targetDir: string, fileName?: string): Promise<void>
|
||||
|
||||
#### Properties
|
||||
// Get a stream that emits StreamFile objects for each file in the archive
|
||||
exportToStreamOfStreamFiles(): Promise<StreamIntake<StreamFile>>
|
||||
|
||||
- `archiveAnalyzer` - Analyze archive contents
|
||||
- `tarTools` - TAR-specific operations
|
||||
- `zipTools` - ZIP-specific operations
|
||||
- `gzipTools` - GZIP-specific operations
|
||||
- `bzip2Tools` - BZIP2-specific operations
|
||||
// Get the raw archive stream (useful for piping)
|
||||
getArchiveStream(): Promise<Readable>
|
||||
```
|
||||
|
||||
### Specialized Tools
|
||||
#### Instance Properties
|
||||
|
||||
Each tool class provides format-specific operations:
|
||||
```typescript
|
||||
archive.tarTools // TarTools instance for TAR-specific operations
|
||||
archive.zipTools // ZipTools instance for ZIP-specific operations
|
||||
archive.gzipTools // GzipTools instance for GZIP-specific operations
|
||||
archive.bzip2Tools // Bzip2Tools instance for BZIP2-specific operations
|
||||
archive.archiveAnalyzer // ArchiveAnalyzer for inspecting archive type
|
||||
```
|
||||
|
||||
- **TarTools** - Pack/unpack TAR archives
|
||||
- **ZipTools** - Handle ZIP compression
|
||||
- **GzipTools** - GZIP compression/decompression
|
||||
- **Bzip2Tools** - BZIP2 operations
|
||||
### TarTools Class
|
||||
|
||||
TAR-specific operations for creating and extracting TAR archives.
|
||||
|
||||
```typescript
|
||||
import { TarTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const tarTools = new TarTools();
|
||||
|
||||
// Get a tar pack stream for creating archives
|
||||
const pack = await tarTools.getPackStream();
|
||||
|
||||
// Add files to a pack stream
|
||||
await tarTools.addFileToPack(pack, {
|
||||
fileName: 'file.txt', // Name in archive
|
||||
content: 'Hello World', // String, Buffer, Readable, SmartFile, or StreamFile
|
||||
byteLength?: number, // Optional: specify size for streams
|
||||
filePath?: string // Optional: path to file on disk
|
||||
});
|
||||
|
||||
// Pack an entire directory
|
||||
const pack = await tarTools.packDirectory('./src');
|
||||
|
||||
// Get extraction stream
|
||||
const extract = tarTools.getDecompressionStream();
|
||||
```
|
||||
|
||||
### ZipTools Class
|
||||
|
||||
ZIP-specific operations.
|
||||
|
||||
```typescript
|
||||
import { ZipTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const zipTools = new ZipTools();
|
||||
|
||||
// Get compression stream (for creating ZIP)
|
||||
const compressor = zipTools.getCompressionStream();
|
||||
|
||||
// Get decompression stream (for extracting ZIP)
|
||||
const decompressor = zipTools.getDecompressionStream();
|
||||
```
|
||||
|
||||
### GzipTools Class
|
||||
|
||||
GZIP compression/decompression streams.
|
||||
|
||||
```typescript
|
||||
import { GzipTools } from '@push.rocks/smartarchive';
|
||||
|
||||
const gzipTools = new GzipTools();
|
||||
|
||||
// Get compression stream
|
||||
const compressor = gzipTools.getCompressionStream();
|
||||
|
||||
// Get decompression stream
|
||||
const decompressor = gzipTools.getDecompressionStream();
|
||||
```
|
||||
|
||||
## Supported Formats 📋
|
||||
|
||||
| Format | Extension(s) | Extract | Create |
|
||||
|--------|--------------|---------|--------|
|
||||
| TAR | `.tar` | ✅ | ✅ |
|
||||
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ⚠️ |
|
||||
| ZIP | `.zip` | ✅ | ⚠️ |
|
||||
| GZIP | `.gz` | ✅ | ✅ |
|
||||
| BZIP2 | `.bz2` | ✅ | ❌ |
|
||||
|
||||
✅ Full support | ⚠️ Partial/basic support | ❌ Not supported
|
||||
|
||||
## Performance Tips 🏎️
|
||||
|
||||
1. **Use streaming for large files** - Avoid loading entire archives into memory
|
||||
2. **Process files in parallel** - Utilize stream operations for concurrent processing
|
||||
3. **Choose the right format** - TAR.GZ for Unix systems, ZIP for cross-platform compatibility
|
||||
4. **Enable compression wisely** - Balance between file size and CPU usage
|
||||
1. **Use streaming for large files** – Avoid loading entire archives into memory with `exportToStreamOfStreamFiles()`
|
||||
2. **Provide byte lengths when known** – When adding streams to TAR, provide `byteLength` for better performance
|
||||
3. **Process files as they stream** – Don't collect all files into an array unless necessary
|
||||
4. **Choose the right format** – TAR.GZ for Unix/compression, ZIP for cross-platform compatibility
|
||||
|
||||
## Error Handling 🛡️
|
||||
|
||||
@@ -295,6 +382,8 @@ try {
|
||||
console.error('Archive file not found');
|
||||
} else if (error.code === 'EACCES') {
|
||||
console.error('Permission denied');
|
||||
} else if (error.message.includes('fetch')) {
|
||||
console.error('Network error downloading archive');
|
||||
} else {
|
||||
console.error('Archive extraction failed:', error.message);
|
||||
}
|
||||
@@ -303,35 +392,57 @@ try {
|
||||
|
||||
## Real-World Use Cases 🌍
|
||||
|
||||
### Backup System
|
||||
```typescript
|
||||
// Automated backup extraction
|
||||
const backup = await SmartArchive.fromArchiveFile('./backup.tar.gz');
|
||||
await backup.exportToFs('/restore/location');
|
||||
```
|
||||
### CI/CD: Download & Extract Build Artifacts
|
||||
|
||||
### CI/CD Pipeline
|
||||
```typescript
|
||||
// Download and extract build artifacts
|
||||
const artifacts = await SmartArchive.fromArchiveUrl(
|
||||
`${CI_SERVER}/artifacts/build-${BUILD_ID}.zip`
|
||||
);
|
||||
await artifacts.exportToFs('./dist');
|
||||
```
|
||||
|
||||
### Data Processing
|
||||
### Backup System: Restore from Archive
|
||||
|
||||
```typescript
|
||||
// Process compressed datasets
|
||||
const dataset = await SmartArchive.fromArchiveUrl(
|
||||
'https://data.source/dataset.tar.bz2'
|
||||
const backup = await SmartArchive.fromArchiveFile('./backup-2024.tar.gz');
|
||||
await backup.exportToFs('/restore/location');
|
||||
```
|
||||
|
||||
### NPM Package Inspection
|
||||
|
||||
```typescript
|
||||
const pkg = await SmartArchive.fromArchiveUrl(
|
||||
'https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz'
|
||||
);
|
||||
const files = await pkg.exportToStreamOfStreamFiles();
|
||||
|
||||
files.on('data', async (file) => {
|
||||
if (file.relativeFilePath.includes('package.json')) {
|
||||
const stream = await file.createReadStream();
|
||||
// Read and analyze package.json
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
### Data Pipeline: Process Compressed Datasets
|
||||
|
||||
```typescript
|
||||
const dataset = await SmartArchive.fromArchiveUrl(
|
||||
'https://data.source/dataset.tar.gz'
|
||||
);
|
||||
|
||||
const files = await dataset.exportToStreamOfStreamFiles();
|
||||
// Process each file in the dataset
|
||||
files.on('data', async (file) => {
|
||||
if (file.relativeFilePath.endsWith('.csv')) {
|
||||
const stream = await file.createReadStream();
|
||||
// Stream CSV processing
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
@@ -341,9 +452,9 @@ This project is owned and maintained by Task Venture Capital GmbH. The names and
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Task Venture Capital GmbH
|
||||
Registered at District court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@@ -1,7 +1,33 @@
|
||||
import * as path from 'path';
|
||||
import * as path from 'node:path';
|
||||
import * as fs from 'node:fs';
|
||||
import * as fsPromises from 'node:fs/promises';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import * as smartrequest from '@push.rocks/smartrequest';
|
||||
import * as smartstream from '@push.rocks/smartstream';
|
||||
|
||||
export { path, smartpath, smartfile, smartrequest, smartstream };
|
||||
export { path, fs, fsPromises, smartpath, smartfile, smartrequest, smartstream };
|
||||
|
||||
/**
|
||||
* List files in a directory recursively, returning relative paths
|
||||
*/
|
||||
export async function listFileTree(dirPath: string, _pattern: string = '**/*'): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
|
||||
async function walkDir(currentPath: string, relativePath: string = '') {
|
||||
const entries = await fsPromises.readdir(currentPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const entryRelPath = relativePath ? path.join(relativePath, entry.name) : entry.name;
|
||||
const entryFullPath = path.join(currentPath, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await walkDir(entryFullPath, entryRelPath);
|
||||
} else if (entry.isFile()) {
|
||||
results.push(entryRelPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await walkDir(dirPath);
|
||||
return results;
|
||||
}
|
||||
|
||||
401
test/test.gzip.node+deno.ts
Normal file
401
test/test.gzip.node+deno.ts
Normal file
@@ -0,0 +1,401 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from './plugins.js';
|
||||
import * as smartarchive from '../ts/index.js';
|
||||
|
||||
const testPaths = {
|
||||
nogitDir: plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../.nogit/',
|
||||
),
|
||||
gzipTestDir: plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../.nogit/gzip-test',
|
||||
),
|
||||
};
|
||||
|
||||
tap.preTask('should prepare test directories', async () => {
|
||||
await plugins.fsPromises.mkdir(testPaths.gzipTestDir, { recursive: true });
|
||||
});
|
||||
|
||||
tap.test('should create and extract a gzip file', async () => {
|
||||
// Create test data
|
||||
const testContent = 'This is a test file for gzip compression and decompression.\n'.repeat(100);
|
||||
const testFileName = 'test-file.txt';
|
||||
const gzipFileName = 'test-file.txt.gz';
|
||||
|
||||
// Write the original file
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, testFileName),
|
||||
testContent
|
||||
);
|
||||
|
||||
// Create gzip compressed version using fflate directly
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||
Buffer.from(compressed)
|
||||
);
|
||||
|
||||
// Now test extraction using SmartArchive
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Export to a new location
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'extracted');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'test-file.txt');
|
||||
|
||||
// Read the extracted file
|
||||
const extractedContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, 'test-file.txt'),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
// Verify the content matches
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
});
|
||||
|
||||
tap.test('should handle gzip stream extraction', async () => {
|
||||
// Create test data
|
||||
const testContent = 'Stream test data for gzip\n'.repeat(50);
|
||||
const gzipFileName = 'stream-test.txt.gz';
|
||||
|
||||
// Create gzip compressed version
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||
Buffer.from(compressed)
|
||||
);
|
||||
|
||||
// Create a read stream for the gzip file
|
||||
const gzipStream = plugins.fs.createReadStream(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction using SmartArchive from stream
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveStream(gzipStream);
|
||||
|
||||
// Export to stream and collect the result
|
||||
const streamFiles: any[] = [];
|
||||
const resultStream = await gzipArchive.exportToStreamOfStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
resultStream.on('data', (streamFile) => {
|
||||
streamFiles.push(streamFile);
|
||||
});
|
||||
resultStream.on('end', resolve);
|
||||
resultStream.on('error', reject);
|
||||
});
|
||||
|
||||
// Verify we got the expected file
|
||||
expect(streamFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Read content from the stream file
|
||||
if (streamFiles[0]) {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await streamFiles[0].createReadStream();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
readStream.on('end', resolve);
|
||||
readStream.on('error', reject);
|
||||
});
|
||||
|
||||
const extractedContent = Buffer.concat(chunks).toString();
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should handle gzip files with original filename in header', async () => {
|
||||
// Test with a real-world gzip file that includes filename in header
|
||||
const testContent = 'File with name in gzip header\n'.repeat(30);
|
||||
const originalFileName = 'original-name.log';
|
||||
const gzipFileName = 'compressed.gz';
|
||||
|
||||
// Create a proper gzip with filename header using Node's zlib
|
||||
const zlib = await import('node:zlib');
|
||||
const gzipBuffer = await new Promise<Buffer>((resolve, reject) => {
|
||||
zlib.gzip(Buffer.from(testContent), {
|
||||
level: 9,
|
||||
// Note: Node's zlib doesn't support embedding filename directly,
|
||||
// but we can test the extraction anyway
|
||||
}, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(result);
|
||||
});
|
||||
});
|
||||
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||
gzipBuffer
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'header-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't reliably contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'compressed.txt');
|
||||
|
||||
// Check if file was extracted (name might be derived from archive name)
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
// Read and verify content
|
||||
const extractedFile = files[0];
|
||||
const extractedContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, extractedFile || 'compressed.txt'),
|
||||
'utf8'
|
||||
);
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
});
|
||||
|
||||
tap.test('should handle large gzip files', async () => {
|
||||
// Create a larger test file
|
||||
const largeContent = 'x'.repeat(1024 * 1024); // 1MB of 'x' characters
|
||||
const gzipFileName = 'large-file.txt.gz';
|
||||
|
||||
// Compress the large file
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(largeContent));
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
|
||||
Buffer.from(compressed)
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'large-extracted');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'large-file.txt');
|
||||
|
||||
// Verify the extracted content
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
const extractedContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, files[0] || 'large-file.txt'),
|
||||
'utf8'
|
||||
);
|
||||
expect(extractedContent.length).toEqual(largeContent.length);
|
||||
expect(extractedContent).toEqual(largeContent);
|
||||
});
|
||||
|
||||
tap.test('should handle real-world multi-chunk gzip from URL', async () => {
|
||||
// Test with a real tgz file that will be processed in multiple chunks
|
||||
const testUrl = 'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz';
|
||||
|
||||
// Download and extract the archive
|
||||
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(testUrl);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'real-world-test');
|
||||
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
|
||||
|
||||
// This will test multi-chunk decompression as the file is larger
|
||||
await testArchive.exportToFs(extractPath);
|
||||
|
||||
// Verify extraction worked
|
||||
const files = await plugins.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
// Check for expected package structure
|
||||
const hasPackageJson = files.some(f => f.includes('package.json'));
|
||||
expect(hasPackageJson).toBeTrue();
|
||||
|
||||
// Read and verify package.json content
|
||||
const packageJsonPath = files.find(f => f.includes('package.json'));
|
||||
if (packageJsonPath) {
|
||||
const packageJsonContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, packageJsonPath),
|
||||
'utf8'
|
||||
);
|
||||
const packageJson = JSON.parse(packageJsonContent);
|
||||
expect(packageJson.name).toEqual('@push.rocks/smartfile');
|
||||
expect(packageJson.version).toEqual('11.2.7');
|
||||
}
|
||||
|
||||
// Read and verify a TypeScript file
|
||||
const tsFilePath = files.find(f => f.endsWith('.ts'));
|
||||
if (tsFilePath) {
|
||||
const tsFileContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, tsFilePath),
|
||||
'utf8'
|
||||
);
|
||||
// TypeScript files should have content
|
||||
expect(tsFileContent.length).toBeGreaterThan(10);
|
||||
console.log(` ✓ TypeScript file ${tsFilePath} has ${tsFileContent.length} bytes`);
|
||||
}
|
||||
|
||||
// Read and verify license file
|
||||
const licensePath = files.find(f => f.includes('license'));
|
||||
if (licensePath) {
|
||||
const licenseContent = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, licensePath),
|
||||
'utf8'
|
||||
);
|
||||
expect(licenseContent).toContain('MIT');
|
||||
}
|
||||
|
||||
// Verify we can read multiple files without corruption
|
||||
const readableFiles = files.filter(f =>
|
||||
f.endsWith('.json') || f.endsWith('.md') || f.endsWith('.ts') || f.endsWith('.js')
|
||||
).slice(0, 5); // Test first 5 readable files
|
||||
|
||||
for (const file of readableFiles) {
|
||||
const content = await plugins.fsPromises.readFile(
|
||||
plugins.path.join(extractPath, file),
|
||||
'utf8'
|
||||
);
|
||||
expect(content).toBeDefined();
|
||||
expect(content.length).toBeGreaterThan(0);
|
||||
console.log(` ✓ Successfully read ${file} (${content.length} bytes)`);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should handle gzip extraction fully in memory', async () => {
|
||||
// Create test data in memory
|
||||
const testContent = 'This is test data for in-memory gzip processing\n'.repeat(100);
|
||||
|
||||
// Compress using fflate in memory
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
|
||||
// Create a stream from the compressed data
|
||||
const { Readable } = await import('node:stream');
|
||||
const compressedStream = Readable.from(Buffer.from(compressed));
|
||||
|
||||
// Process through SmartArchive without touching filesystem
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveStream(compressedStream);
|
||||
|
||||
// Export to stream of stream files (in memory)
|
||||
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||
const resultStream = await gzipArchive.exportToStreamOfStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||
streamFiles.push(streamFile);
|
||||
});
|
||||
resultStream.on('end', resolve);
|
||||
resultStream.on('error', reject);
|
||||
});
|
||||
|
||||
// Verify we got a file
|
||||
expect(streamFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Read the content from memory without filesystem
|
||||
const firstFile = streamFiles[0];
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await firstFile.createReadStream();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
readStream.on('end', resolve);
|
||||
readStream.on('error', reject);
|
||||
});
|
||||
|
||||
const extractedContent = Buffer.concat(chunks).toString();
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
console.log(` ✓ In-memory extraction successful (${extractedContent.length} bytes)`);
|
||||
});
|
||||
|
||||
tap.test('should handle real tgz file fully in memory', async (tools) => {
|
||||
await tools.timeout(10000); // Set 10 second timeout
|
||||
// Download tgz file into memory
|
||||
const response = await plugins.smartrequest.SmartRequest.create()
|
||||
.url('https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz')
|
||||
.get();
|
||||
|
||||
const tgzBuffer = Buffer.from(await response.arrayBuffer());
|
||||
console.log(` Downloaded ${tgzBuffer.length} bytes into memory`);
|
||||
|
||||
// Create stream from buffer
|
||||
const { Readable: Readable2 } = await import('node:stream');
|
||||
const tgzStream = Readable2.from(tgzBuffer);
|
||||
|
||||
// Process through SmartArchive in memory
|
||||
const archive = await smartarchive.SmartArchive.fromArchiveStream(tgzStream);
|
||||
|
||||
// Export to stream of stream files (in memory)
|
||||
const streamFiles: plugins.smartfile.StreamFile[] = [];
|
||||
const resultStream = await archive.exportToStreamOfStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let timeout: NodeJS.Timeout;
|
||||
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
};
|
||||
|
||||
timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
resolve(); // Resolve after timeout if stream doesn't end
|
||||
}, 5000);
|
||||
|
||||
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
|
||||
streamFiles.push(streamFile);
|
||||
});
|
||||
|
||||
resultStream.on('end', () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
});
|
||||
|
||||
resultStream.on('error', (err) => {
|
||||
cleanup();
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
|
||||
console.log(` Extracted ${streamFiles.length} files in memory`);
|
||||
// At minimum we should have extracted something
|
||||
expect(streamFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Find and read package.json from memory
|
||||
const packageJsonFile = streamFiles.find(f => f.relativeFilePath?.includes('package.json'));
|
||||
|
||||
if (packageJsonFile) {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await packageJsonFile.createReadStream();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
readStream.on('end', resolve);
|
||||
readStream.on('error', reject);
|
||||
});
|
||||
|
||||
const packageJsonContent = Buffer.concat(chunks).toString();
|
||||
const packageJson = JSON.parse(packageJsonContent);
|
||||
expect(packageJson.name).toEqual('@push.rocks/smartfile');
|
||||
expect(packageJson.version).toEqual('11.2.7');
|
||||
console.log(` ✓ Read package.json from memory: ${packageJson.name}@${packageJson.version}`);
|
||||
}
|
||||
|
||||
// Read a few more files to verify integrity
|
||||
const filesToCheck = streamFiles.slice(0, 3);
|
||||
for (const file of filesToCheck) {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await file.createReadStream();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
readStream.on('end', resolve);
|
||||
readStream.on('error', reject);
|
||||
});
|
||||
|
||||
const content = Buffer.concat(chunks);
|
||||
expect(content.length).toBeGreaterThan(0);
|
||||
console.log(` ✓ Read ${file.relativeFilePath} from memory (${content.length} bytes)`);
|
||||
}
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -1,219 +0,0 @@
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from './plugins.js';
|
||||
import * as smartarchive from '../ts/index.js';
|
||||
|
||||
const testPaths = {
|
||||
nogitDir: plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../.nogit/',
|
||||
),
|
||||
gzipTestDir: plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../.nogit/gzip-test',
|
||||
),
|
||||
};
|
||||
|
||||
tap.preTask('should prepare test directories', async () => {
|
||||
await plugins.smartfile.fs.ensureDir(testPaths.gzipTestDir);
|
||||
});
|
||||
|
||||
tap.test('should create and extract a gzip file', async () => {
|
||||
// Create test data
|
||||
const testContent = 'This is a test file for gzip compression and decompression.\n'.repeat(100);
|
||||
const testFileName = 'test-file.txt';
|
||||
const gzipFileName = 'test-file.txt.gz';
|
||||
|
||||
// Write the original file
|
||||
await plugins.smartfile.memory.toFs(
|
||||
testContent,
|
||||
plugins.path.join(testPaths.gzipTestDir, testFileName)
|
||||
);
|
||||
|
||||
// Compress the file using gzip
|
||||
const originalFile = await plugins.smartfile.fs.fileTreeToObject(
|
||||
testPaths.gzipTestDir,
|
||||
testFileName
|
||||
);
|
||||
|
||||
// Create gzip compressed version using fflate directly
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
await plugins.smartfile.memory.toFs(
|
||||
Buffer.from(compressed),
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Now test extraction using SmartArchive
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Export to a new location
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'extracted');
|
||||
await plugins.smartfile.fs.ensureDir(extractPath);
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'test-file.txt');
|
||||
|
||||
// Read the extracted file
|
||||
const extractedContent = await plugins.smartfile.fs.toStringSync(
|
||||
plugins.path.join(extractPath, 'test-file.txt')
|
||||
);
|
||||
|
||||
// Verify the content matches
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
});
|
||||
|
||||
tap.test('should handle gzip stream extraction', async () => {
|
||||
// Create test data
|
||||
const testContent = 'Stream test data for gzip\n'.repeat(50);
|
||||
const gzipFileName = 'stream-test.txt.gz';
|
||||
|
||||
// Create gzip compressed version
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(testContent));
|
||||
await plugins.smartfile.memory.toFs(
|
||||
Buffer.from(compressed),
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Create a read stream for the gzip file
|
||||
const gzipStream = plugins.smartfile.fsStream.createReadStream(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction using SmartArchive from stream
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveStream(gzipStream);
|
||||
|
||||
// Export to stream and collect the result
|
||||
const streamFiles: any[] = [];
|
||||
const resultStream = await gzipArchive.exportToStreamOfStreamFiles();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
resultStream.on('data', (streamFile) => {
|
||||
streamFiles.push(streamFile);
|
||||
});
|
||||
resultStream.on('end', resolve);
|
||||
resultStream.on('error', reject);
|
||||
});
|
||||
|
||||
// Verify we got the expected file
|
||||
expect(streamFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Read content from the stream file
|
||||
if (streamFiles[0]) {
|
||||
const chunks: Buffer[] = [];
|
||||
const readStream = await streamFiles[0].createReadStream();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
|
||||
readStream.on('end', resolve);
|
||||
readStream.on('error', reject);
|
||||
});
|
||||
|
||||
const extractedContent = Buffer.concat(chunks).toString();
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('should handle gzip files with original filename in header', async () => {
|
||||
// Test with a real-world gzip file that includes filename in header
|
||||
const testContent = 'File with name in gzip header\n'.repeat(30);
|
||||
const originalFileName = 'original-name.log';
|
||||
const gzipFileName = 'compressed.gz';
|
||||
|
||||
// Create a proper gzip with filename header using Node's zlib
|
||||
const zlib = await import('zlib');
|
||||
const gzipBuffer = await new Promise<Buffer>((resolve, reject) => {
|
||||
zlib.gzip(Buffer.from(testContent), {
|
||||
level: 9,
|
||||
// Note: Node's zlib doesn't support embedding filename directly,
|
||||
// but we can test the extraction anyway
|
||||
}, (err, result) => {
|
||||
if (err) reject(err);
|
||||
else resolve(result);
|
||||
});
|
||||
});
|
||||
|
||||
await plugins.smartfile.memory.toFs(
|
||||
gzipBuffer,
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'header-test');
|
||||
await plugins.smartfile.fs.ensureDir(extractPath);
|
||||
// Provide a filename since gzip doesn't reliably contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'compressed.txt');
|
||||
|
||||
// Check if file was extracted (name might be derived from archive name)
|
||||
const files = await plugins.smartfile.fs.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
// Read and verify content
|
||||
const extractedFile = files[0];
|
||||
const extractedContent = await plugins.smartfile.fs.toStringSync(
|
||||
plugins.path.join(extractPath, extractedFile || 'compressed.txt')
|
||||
);
|
||||
expect(extractedContent).toEqual(testContent);
|
||||
});
|
||||
|
||||
tap.test('should handle large gzip files', async () => {
|
||||
// Create a larger test file
|
||||
const largeContent = 'x'.repeat(1024 * 1024); // 1MB of 'x' characters
|
||||
const gzipFileName = 'large-file.txt.gz';
|
||||
|
||||
// Compress the large file
|
||||
const fflate = await import('fflate');
|
||||
const compressed = fflate.gzipSync(Buffer.from(largeContent));
|
||||
await plugins.smartfile.memory.toFs(
|
||||
Buffer.from(compressed),
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
// Test extraction
|
||||
const gzipArchive = await smartarchive.SmartArchive.fromArchiveFile(
|
||||
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
|
||||
);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'large-extracted');
|
||||
await plugins.smartfile.fs.ensureDir(extractPath);
|
||||
// Provide a filename since gzip doesn't contain filename metadata
|
||||
await gzipArchive.exportToFs(extractPath, 'large-file.txt');
|
||||
|
||||
// Verify the extracted content
|
||||
const files = await plugins.smartfile.fs.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
const extractedContent = await plugins.smartfile.fs.toStringSync(
|
||||
plugins.path.join(extractPath, files[0] || 'large-file.txt')
|
||||
);
|
||||
expect(extractedContent.length).toEqual(largeContent.length);
|
||||
expect(extractedContent).toEqual(largeContent);
|
||||
});
|
||||
|
||||
tap.test('should handle real-world multi-chunk gzip from URL', async () => {
|
||||
// Test with a real tgz file that will be processed in multiple chunks
|
||||
const testUrl = 'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz';
|
||||
|
||||
// Download and extract the archive
|
||||
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(testUrl);
|
||||
|
||||
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'real-world-test');
|
||||
await plugins.smartfile.fs.ensureDir(extractPath);
|
||||
|
||||
// This will test multi-chunk decompression as the file is larger
|
||||
await testArchive.exportToFs(extractPath);
|
||||
|
||||
// Verify extraction worked
|
||||
const files = await plugins.smartfile.fs.listFileTree(extractPath, '**/*');
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
|
||||
// Check for expected package structure
|
||||
const hasPackageJson = files.some(f => f.includes('package.json'));
|
||||
expect(hasPackageJson).toBeTrue();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -16,7 +16,7 @@ const testPaths = {
|
||||
import * as smartarchive from '../ts/index.js';
|
||||
|
||||
tap.preTask('should prepare .nogit dir', async () => {
|
||||
await plugins.smartfile.fs.ensureDir(testPaths.remoteDir);
|
||||
await plugins.fsPromises.mkdir(testPaths.remoteDir, { recursive: true });
|
||||
});
|
||||
|
||||
tap.preTask('should prepare downloads', async (tools) => {
|
||||
@@ -26,9 +26,9 @@ tap.preTask('should prepare downloads', async (tools) => {
|
||||
)
|
||||
.get();
|
||||
const downloadedFile: Buffer = Buffer.from(await response.arrayBuffer());
|
||||
await plugins.smartfile.memory.toFs(
|
||||
downloadedFile,
|
||||
await plugins.fsPromises.writeFile(
|
||||
plugins.path.join(testPaths.nogitDir, 'test.tgz'),
|
||||
downloadedFile,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartarchive',
|
||||
version: '4.2.1',
|
||||
version: '4.2.4',
|
||||
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ export class SmartArchive {
|
||||
return urlStream;
|
||||
}
|
||||
if (this.sourceFilePath) {
|
||||
const fileStream = plugins.smartfile.fs.toReadStream(this.sourceFilePath);
|
||||
const fileStream = plugins.fs.createReadStream(this.sourceFilePath);
|
||||
return fileStream;
|
||||
}
|
||||
}
|
||||
@@ -116,14 +116,13 @@ export class SmartArchive {
|
||||
);
|
||||
const streamFile = streamFileArg;
|
||||
const readStream = await streamFile.createReadStream();
|
||||
await plugins.smartfile.fs.ensureDir(targetDir);
|
||||
await plugins.fsPromises.mkdir(targetDir, { recursive: true });
|
||||
const writePath = plugins.path.join(
|
||||
targetDir,
|
||||
streamFile.relativeFilePath || fileNameArg,
|
||||
);
|
||||
await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath));
|
||||
const writeStream =
|
||||
plugins.smartfile.fsStream.createWriteStream(writePath);
|
||||
await plugins.fsPromises.mkdir(plugins.path.dirname(writePath), { recursive: true });
|
||||
const writeStream = plugins.fs.createWriteStream(writePath);
|
||||
readStream.pipe(writeStream);
|
||||
writeStream.on('finish', () => {
|
||||
done.resolve();
|
||||
@@ -158,21 +157,34 @@ export class SmartArchive {
|
||||
console.log(
|
||||
`tar stream directory: ${header.name} ... skipping!`,
|
||||
);
|
||||
next();
|
||||
stream.resume(); // Consume directory stream
|
||||
stream.on('end', () => next());
|
||||
return;
|
||||
}
|
||||
console.log(`tar stream file: ${header.name}`);
|
||||
|
||||
// Create a PassThrough stream to buffer the data
|
||||
const passThrough = new plugins.stream.PassThrough();
|
||||
const streamfile = plugins.smartfile.StreamFile.fromStream(
|
||||
stream,
|
||||
passThrough,
|
||||
header.name,
|
||||
);
|
||||
|
||||
// Push the streamfile immediately
|
||||
streamFileIntake.push(streamfile);
|
||||
stream.on('end', function () {
|
||||
next(); // ready for next entry
|
||||
|
||||
// Pipe the tar entry stream to the passthrough
|
||||
stream.pipe(passThrough);
|
||||
|
||||
// Move to next entry when this one ends
|
||||
stream.on('end', () => {
|
||||
passThrough.end();
|
||||
next();
|
||||
});
|
||||
});
|
||||
tarStream.on('finish', function () {
|
||||
console.log('finished');
|
||||
console.log('tar extraction finished');
|
||||
// Only signal end if this is the final stream
|
||||
streamFileIntake.signalEnd();
|
||||
});
|
||||
analyzedResultChunk.resultStream.pipe(
|
||||
@@ -199,10 +211,13 @@ export class SmartArchive {
|
||||
analyzedResultChunk.isArchive &&
|
||||
analyzedResultChunk.decompressionStream
|
||||
) {
|
||||
analyzedResultChunk.resultStream
|
||||
// For nested archives (like gzip containing tar)
|
||||
const nestedStream = analyzedResultChunk.resultStream
|
||||
.pipe(analyzedResultChunk.decompressionStream)
|
||||
.pipe(createAnalyzedStream())
|
||||
.pipe(createUnpackStream());
|
||||
|
||||
// Don't signal end here - let the nested unpacker handle it
|
||||
} else {
|
||||
const streamFile = plugins.smartfile.StreamFile.fromStream(
|
||||
analyzedResultChunk.resultStream,
|
||||
|
||||
@@ -55,7 +55,7 @@ export class TarTools {
|
||||
'@push.rocks/smartarchive: When streaming, it is recommended to provide byteLength, if known.',
|
||||
);
|
||||
} else if (optionsArg.filePath) {
|
||||
const fileStat = await plugins.smartfile.fs.stat(optionsArg.filePath);
|
||||
const fileStat = await plugins.fsPromises.stat(optionsArg.filePath);
|
||||
contentByteLength = fileStat.size;
|
||||
}
|
||||
|
||||
@@ -109,19 +109,16 @@ export class TarTools {
|
||||
* @param directoryPath
|
||||
*/
|
||||
public async packDirectory(directoryPath: string) {
|
||||
const fileTree = await plugins.smartfile.fs.listFileTree(
|
||||
directoryPath,
|
||||
'**/*',
|
||||
);
|
||||
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
|
||||
const pack = await this.getPackStream();
|
||||
for (const filePath of fileTree) {
|
||||
const absolutePath = plugins.path.join(directoryPath, filePath);
|
||||
const fileStat = await plugins.smartfile.fs.stat(absolutePath);
|
||||
const fileStat = await plugins.fsPromises.stat(absolutePath);
|
||||
await this.addFileToPack(pack, {
|
||||
byteLength: fileStat.size,
|
||||
filePath: absolutePath,
|
||||
fileName: filePath,
|
||||
content: plugins.smartfile.fsStream.createReadStream(absolutePath),
|
||||
content: plugins.fs.createReadStream(absolutePath),
|
||||
});
|
||||
}
|
||||
return pack;
|
||||
|
||||
@@ -1,8 +1,34 @@
|
||||
// node native scope
|
||||
import * as path from 'path';
|
||||
import * as stream from 'stream';
|
||||
import * as path from 'node:path';
|
||||
import * as stream from 'node:stream';
|
||||
import * as fs from 'node:fs';
|
||||
import * as fsPromises from 'node:fs/promises';
|
||||
|
||||
export { path, stream };
|
||||
export { path, stream, fs, fsPromises };
|
||||
|
||||
/**
|
||||
* List files in a directory recursively, returning relative paths
|
||||
*/
|
||||
export async function listFileTree(dirPath: string, _pattern: string = '**/*'): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
|
||||
async function walkDir(currentPath: string, relativePath: string = '') {
|
||||
const entries = await fsPromises.readdir(currentPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const entryRelPath = relativePath ? path.join(relativePath, entry.name) : entry.name;
|
||||
const entryFullPath = path.join(currentPath, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await walkDir(entryFullPath, entryRelPath);
|
||||
} else if (entry.isFile()) {
|
||||
results.push(entryRelPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await walkDir(dirPath);
|
||||
return results;
|
||||
}
|
||||
|
||||
// @pushrocks scope
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
|
||||
Reference in New Issue
Block a user