push.rocks/smartarchive
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tartools): add streaming TAR support (tar-stream), Node.js streaming APIs for TarTools, and browser / web bundle docs

Browse Source
This commit is contained in:
Jürgen Kunz
2026-01-01 23:40:13 +00:00
parent d97e9c1dce
commit db48fcd455
13 changed files with 456 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
changelog.md
View File

@@ -1,5 +1,15 @@
# Changelog
## 2026-01-01 - 5.2.0 - feat(tartools)
add streaming TAR support (tar-stream), Node.js streaming APIs for TarTools, and browser / web bundle docs
- Add tar-stream runtime dependency and @types/tar-stream devDependency
- Introduce streaming TarTools APIs: getPackStream, addFileToPack, getExtractStream, extractToDirectory, getDirectoryPackStream, getDirectoryPackStreamGz
- Switch SmartArchive TAR extraction to use tar-stream extract for true streaming ingestion of entries
- Export tarStream in plugins and export ITarPackFileOptions from the Node.js entrypoint
- Update packDirectory/packDirectoryToTarGz to handle files safely and use fflate.gzipSync for buffer-based gzipping
- README updates: document /web browser bundle, browser usage examples, Uint8Array guidance, updated feature table and streaming examples
## 2026-01-01 - 5.1.0 - feat(archive)
introduce ts_shared browser-compatible layer, refactor Node-specific tools to wrap/shared implementations, and modernize archive handling

8
deno.lock generated
View File

@@ -13,9 +13,11 @@
"npm:@push.rocks/smartstream@^3.2.5": "3.2.5",
"npm:@push.rocks/smartunique@^3.0.9": "3.0.9",
"npm:@push.rocks/smarturl@^3.1.0": "3.1.0",
"npm:@types/tar-stream@^3.1.3": "3.1.4",
"npm:fflate@~0.8.2": "0.8.2",
"npm:file-type@^21.2.0": "21.2.0",
"npm:modern-tar@~0.7.3": "0.7.3"
"npm:modern-tar@~0.7.3": "0.7.3",
"npm:tar-stream@^3.1.7": "3.1.7"
},
"npm": {
"@api.global/typedrequest-interfaces@2.0.2": {
@@ -6756,9 +6758,11 @@
"npm:@push.rocks/smartstream@^3.2.5",
"npm:@push.rocks/smartunique@^3.0.9",
"npm:@push.rocks/smarturl@^3.1.0",
"npm:@types/tar-stream@^3.1.3",
"npm:fflate@~0.8.2",
"npm:file-type@^21.2.0",
"npm:modern-tar@~0.7.3"
"npm:modern-tar@~0.7.3",
"npm:tar-stream@^3.1.7"
]
}
}

2
dist_ts/index.d.ts vendored
View File

@@ -1,4 +1,4 @@
export * from '../ts_shared/index.js';
export * from './classes.smartarchive.js';
export * from './classes.archiveanalyzer.js';
export { TarTools } from './classes.tartools.js';
export { TarTools, type ITarPackFileOptions } from './classes.tartools.js';

4
dist_ts/index.js
View File

@@ -4,6 +4,6 @@ export * from '../ts_shared/index.js';
export * from './classes.smartarchive.js';
// Node.js-specific: Archive analysis with SmartArchive integration
export * from './classes.archiveanalyzer.js';
// Node.js-specific: Extended TarTools with filesystem support (overrides shared TarTools)
// Node.js-specific: Extended TarTools with streaming support (overrides shared TarTools)
export { TarTools } from './classes.tartools.js';
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSwyREFBMkQ7QUFDM0QsY0FBYyx1QkFBdUIsQ0FBQztBQUV0QywrREFBK0Q7QUFDL0QsY0FBYywyQkFBMkIsQ0FBQztBQUUxQyxtRUFBbUU7QUFDbkUsY0FBYyw4QkFBOEIsQ0FBQztBQUU3QywwRkFBMEY7QUFDMUYsT0FBTyxFQUFFLFFBQVEsRUFBRSxNQUFNLHVCQUF1QixDQUFDIn0=
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi90cy9pbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSwyREFBMkQ7QUFDM0QsY0FBYyx1QkFBdUIsQ0FBQztBQUV0QywrREFBK0Q7QUFDL0QsY0FBYywyQkFBMkIsQ0FBQztBQUUxQyxtRUFBbUU7QUFDbkUsY0FBYyw4QkFBOEIsQ0FBQztBQUU3Qyx5RkFBeUY7QUFDekYsT0FBTyxFQUFFLFFBQVEsRUFBNEIsTUFBTSx1QkFBdUIsQ0FBQyJ9

6
package.json
View File

@@ -36,12 +36,14 @@
"@push.rocks/smarturl": "^3.1.0",
"fflate": "^0.8.2",
"file-type": "^21.2.0",
"modern-tar": "^0.7.3"
"modern-tar": "^0.7.3",
"tar-stream": "^3.1.7"
},
"devDependencies": {
"@git.zone/tsbuild": "^4.0.2",
"@git.zone/tsrun": "^2.0.1",
"@git.zone/tstest": "^3.1.4"
"@git.zone/tstest": "^3.1.4",
"@types/tar-stream": "^3.1.3"
},
"private": false,
"files": [

6
pnpm-lock.yaml generated
View File

@@ -44,6 +44,9 @@ importers:
modern-tar:
specifier: ^0.7.3
version: 0.7.3
tar-stream:
specifier: ^3.1.7
version: 3.1.7
devDependencies:
'@git.zone/tsbuild':
specifier: ^4.0.2
@@ -54,6 +57,9 @@ importers:
'@git.zone/tstest':
specifier: ^3.1.4
version: 3.1.4(socks@2.8.7)(typescript@5.9.3)
'@types/tar-stream':
specifier: ^3.1.3
version: 3.1.4
packages:

202
readme.md
View File

@@ -1,6 +1,6 @@
# @push.rocks/smartarchive 📦
A powerful, streaming-first archive manipulation library with a fluent builder API. Works seamlessly in Node.js and Deno.
A powerful, streaming-first archive manipulation library with a fluent builder API. Works seamlessly in **Node.js**, **Deno**, and **browsers**.
## Issue Reporting and Security
@@ -12,11 +12,12 @@ For reporting bugs, issues, or security vulnerabilities, please visit [community
- 🌊 **Streaming-first architecture** – Process large archives without memory constraints
- ✨ **Fluent builder API** – Chain methods for readable, expressive code
- 🎯 **Smart detection** – Automatically identifies archive types via magic bytes
- ⚡ **High performance** – Built on `tar-stream` and `fflate` for speed
- ⚡ **High performance** – Built on `modern-tar` and `fflate` for speed
- 🔧 **Flexible I/O** – Work with files, URLs, streams, and buffers seamlessly
- 🛠️ **Modern TypeScript** – Full type safety and excellent IDE support
- 🔄 **Dual-mode operation** – Extract existing archives OR create new ones
- 🦕 **Cross-runtime** – Works in both Node.js and Deno environments
- 🦕 **Cross-runtime** – Works in Node.js, Deno, and browsers
- 🌐 **Browser-ready** – Dedicated browser bundle with zero Node.js dependencies
## Installation 📥
@@ -71,6 +72,59 @@ await SmartArchive.create()
.extract('./node_modules/lodash');
```
## Browser Usage 🌐
smartarchive provides a dedicated browser-compatible bundle with no Node.js dependencies:
```typescript
// Import from the /web subpath for browser environments
import { TarTools, ZipTools, GzipTools, Bzip2Tools } from '@push.rocks/smartarchive/web';
// Create a TAR archive in the browser
const tarTools = new TarTools();
const tarBuffer = await tarTools.packFiles([
{ archivePath: 'hello.txt', content: 'Hello from the browser!' },
{ archivePath: 'data.json', content: JSON.stringify({ browser: true }) }
]);
// Create a TAR.GZ archive
const tgzBuffer = await tarTools.packFilesToTarGz([
{ archivePath: 'file.txt', content: 'Compressed!' }
], 6);
// Extract a TAR archive
const entries = await tarTools.extractTar(tarBuffer);
for (const entry of entries) {
console.log(`${entry.path}: ${entry.content.length} bytes`);
}
// Work with ZIP files
const zipTools = new ZipTools();
const zipBuffer = await zipTools.createZip([
{ archivePath: 'doc.txt', content: 'Document content' }
], 6);
const zipEntries = await zipTools.extractZip(zipBuffer);
// GZIP compression
const gzipTools = new GzipTools();
const compressed = gzipTools.compressSync(new TextEncoder().encode('Hello World'), 6);
const decompressed = gzipTools.decompressSync(compressed);
```
### Browser Bundle Exports
The `/web` subpath exports these browser-compatible tools:
| Export | Description |
|--------|-------------|
| `TarTools` | Create and extract TAR and TAR.GZ archives |
| `ZipTools` | Create and extract ZIP archives |
| `GzipTools` | GZIP compression and decompression |
| `Bzip2Tools` | BZIP2 decompression (extraction only) |
> 💡 **Note:** The browser bundle does **not** include `SmartArchive` (which requires filesystem access). Use the individual tool classes for browser applications.
## Core Concepts 💡
### Fluent Builder Pattern
@@ -294,21 +348,14 @@ await SmartArchive.create()
// Use GzipTools directly for compression/decompression
const gzipTools = new GzipTools();
// Compress a buffer
const compressed = await gzipTools.compress(Buffer.from('Hello World'), 9);
const decompressed = await gzipTools.decompress(compressed);
// Compress a buffer (sync and async available)
const input = new TextEncoder().encode('Hello World');
const compressed = gzipTools.compressSync(input, 9);
const decompressed = gzipTools.decompressSync(compressed);
// Synchronous operations
const compressedSync = gzipTools.compressSync(inputBuffer, 6);
const decompressedSync = gzipTools.decompressSync(compressedSync);
// Streaming
const compressStream = gzipTools.getCompressionStream(6);
const decompressStream = gzipTools.getDecompressionStream();
createReadStream('./input.txt')
.pipe(compressStream)
.pipe(createWriteStream('./output.gz'));
// Async versions (internally use sync for cross-runtime compatibility)
const compressedAsync = await gzipTools.compress(input, 6);
const decompressedAsync = await gzipTools.decompress(compressedAsync);
```
### Working with TAR archives directly
@@ -318,27 +365,90 @@ import { TarTools } from '@push.rocks/smartarchive';
const tarTools = new TarTools();
// Create a TAR archive manually
const pack = await tarTools.getPackStream();
// Create a TAR archive from entries (buffer-based, good for small files)
const tarBuffer = await tarTools.packFiles([
{ archivePath: 'hello.txt', content: 'Hello, World!' },
{ archivePath: 'data.json', content: JSON.stringify({ foo: 'bar' }) }
]);
// Create a TAR.GZ archive
const tgzBuffer = await tarTools.packFilesToTarGz([
{ archivePath: 'file.txt', content: 'Compressed content' }
], 6);
// Extract a TAR archive
const entries = await tarTools.extractTar(tarBuffer);
for (const entry of entries) {
console.log(`${entry.path}: ${entry.isDirectory ? 'dir' : 'file'}`);
}
// Extract a TAR.GZ archive
const tgzEntries = await tarTools.extractTarGz(tgzBuffer);
// Node.js only: Pack a directory (buffer-based)
const dirBuffer = await tarTools.packDirectory('./src');
const dirTgzBuffer = await tarTools.packDirectoryToTarGz('./src', 9);
```
### Streaming TAR for Large Files (Node.js only) 🚀
For large files that don't fit in memory, use the streaming APIs:
```typescript
import { TarTools } from '@push.rocks/smartarchive';
import * as fs from 'fs';
const tarTools = new TarTools();
// ===== STREAMING PACK =====
// Create a TAR pack stream - files are processed one at a time
const pack = tarTools.getPackStream();
// Add files with streaming content (requires size for streams)
await tarTools.addFileToPack(pack, {
fileName: 'hello.txt',
content: 'Hello, World!'
fileName: 'small.txt',
content: 'Hello World' // Strings and buffers auto-detect size
});
await tarTools.addFileToPack(pack, {
fileName: 'data.json',
content: Buffer.from(JSON.stringify({ foo: 'bar' }))
fileName: 'large-video.mp4',
content: fs.createReadStream('./video.mp4'),
size: fs.statSync('./video.mp4').size // Size required for streams
});
pack.finalize();
pack.pipe(createWriteStream('./output.tar'));
pack.pipe(fs.createWriteStream('output.tar'));
// Pack a directory to TAR.GZ buffer
const tgzBuffer = await tarTools.packDirectoryToTarGz('./src', 6);
// ===== STREAMING DIRECTORY PACK =====
// Pack entire directory with true streaming (no buffering)
const tarStream = await tarTools.getDirectoryPackStream('./large-folder');
tarStream.pipe(fs.createWriteStream('backup.tar'));
// Pack a directory to TAR.GZ stream
const tgzStream = await tarTools.packDirectoryToTarGzStream('./src');
// With GZIP compression
const tgzStream = await tarTools.getDirectoryPackStreamGz('./large-folder', 6);
tgzStream.pipe(fs.createWriteStream('backup.tar.gz'));
// ===== STREAMING EXTRACT =====
// Extract large archives without loading into memory
const extract = tarTools.getExtractStream();
extract.on('entry', (header, stream, next) => {
console.log(`Extracting: ${header.name} (${header.size} bytes)`);
const writeStream = fs.createWriteStream(`./out/${header.name}`);
stream.pipe(writeStream);
writeStream.on('finish', next);
});
extract.on('finish', () => console.log('Extraction complete'));
fs.createReadStream('large-archive.tar').pipe(extract);
// Or use the convenient directory extraction
await tarTools.extractToDirectory(
fs.createReadStream('archive.tar'),
'./output-folder'
);
```
### Working with ZIP archives directly
@@ -351,7 +461,7 @@ const zipTools = new ZipTools();
// Create a ZIP archive from entries
const zipBuffer = await zipTools.createZip([
{ archivePath: 'readme.txt', content: 'Hello!' },
{ archivePath: 'data.bin', content: Buffer.from([0x00, 0x01, 0x02]) }
{ archivePath: 'data.bin', content: new Uint8Array([0x00, 0x01, 0x02]) }
], 6);
// Extract a ZIP buffer
@@ -448,13 +558,13 @@ fileStream.on('data', async (file) => {
## Supported Formats 📋
| Format | Extension(s) | Extract | Create |
|--------|--------------|---------|--------|
| TAR | `.tar` | ✅ | ✅ |
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ✅ |
| ZIP | `.zip` | ✅ | ✅ |
| GZIP | `.gz` | ✅ | ✅ |
| BZIP2 | `.bz2` | ✅ | ❌ |
| Format | Extension(s) | Extract | Create | Browser |
|--------|--------------|---------|--------|---------|
| TAR | `.tar` | ✅ | ✅ | ✅ |
| TAR.GZ / TGZ | `.tar.gz`, `.tgz` | ✅ | ✅ | ✅ |
| ZIP | `.zip` | ✅ | ✅ | ✅ |
| GZIP | `.gz` | ✅ | ✅ | ✅ |
| BZIP2 | `.bz2` | ✅ | ❌ | ✅ |
## Type Definitions
@@ -468,7 +578,7 @@ type TCompressionLevel = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
// Entry for creating archives
interface IArchiveEntry {
archivePath: string;
content: string | Buffer | Readable | SmartFile | StreamFile;
content: string | Buffer | Uint8Array | SmartFile | StreamFile;
size?: number;
mode?: number;
mtime?: Date;
@@ -496,9 +606,9 @@ interface IArchiveInfo {
## Performance Tips 🏎️
1. **Use streaming for large files** – `.toStreamFiles()` processes entries one at a time without loading the entire archive
2. **Provide byte lengths when known** – When using TarTools directly, provide `byteLength` for better performance
3. **Choose appropriate compression** – Use 1-3 for speed, 6 (default) for balance, 9 for maximum compression
4. **Filter early** – Use `.include()`/`.exclude()` to skip unwanted entries before processing
2. **Choose appropriate compression** – Use 1-3 for speed, 6 (default) for balance, 9 for maximum compression
3. **Filter early** – Use `.include()`/`.exclude()` to skip unwanted entries before processing
4. **Use Uint8Array in browsers** – The browser bundle works with `Uint8Array` for optimal performance
## Error Handling 🛡️
@@ -524,23 +634,21 @@ try {
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
### Trademarks
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
### Issue Reporting and Security
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
### Company Information
Task Venture Capital GmbH
Registered at District court Bremen HRB 35230 HB, Germany
Registered at District Court Bremen HRB 35230 HB, Germany
For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
For any legal inquiries or further information, please contact us via email at hello@task.vc.
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

2
ts/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartarchive',
version: '5.1.0',
version: '5.2.0',
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
}

53
ts/classes.smartarchive.ts
View File

@@ -380,36 +380,41 @@ export class SmartArchive {
plugins.smartstream.createTransformFunction<IAnalyzedResult, void>(
async (analyzedResultChunk) => {
if (analyzedResultChunk.fileType?.mime === 'application/x-tar') {
// Use modern-tar for TAR extraction
const chunks: Buffer[] = [];
// Use tar-stream for streaming TAR extraction
// Buffer each entry to ensure tar-stream can proceed to next entry
const extract = this.tarTools.getExtractStream();
analyzedResultChunk.resultStream.on('data', (chunk: Buffer) => {
chunks.push(chunk);
});
analyzedResultChunk.resultStream.on('end', async () => {
try {
const tarBuffer = Buffer.concat(chunks);
const entries = await this.tarTools.extractTar(new Uint8Array(tarBuffer));
for (const entry of entries) {
if (entry.isDirectory) continue;
const streamFile = plugins.smartfile.StreamFile.fromBuffer(
Buffer.from(entry.content)
);
streamFile.relativeFilePath = entry.path;
streamFileIntake.push(streamFile);
}
safeSignalEnd();
} catch (err) {
streamFileIntake.emit('error', err);
extract.on('entry', (header, stream, next) => {
if (header.type === 'directory') {
stream.resume(); // Drain the stream
next();
return;
}
// Buffer the entry content to avoid blocking tar-stream
const chunks: Buffer[] = [];
stream.on('data', (chunk: Buffer) => chunks.push(chunk));
stream.on('end', () => {
const content = Buffer.concat(chunks);
const streamFile = plugins.smartfile.StreamFile.fromBuffer(content);
streamFile.relativeFilePath = header.name;
streamFileIntake.push(streamFile);
next();
});
stream.on('error', (err: Error) => {
streamFileIntake.emit('error', err);
});
});
analyzedResultChunk.resultStream.on('error', (err: Error) => {
extract.on('finish', () => {
safeSignalEnd();
});
extract.on('error', (err: Error) => {
streamFileIntake.emit('error', err);
});
analyzedResultChunk.resultStream.pipe(extract);
} else if (analyzedResultChunk.fileType?.mime === 'application/zip') {
analyzedResultChunk.resultStream
.pipe(analyzedResultChunk.decompressionStream)

249
ts/classes.tartools.ts
View File

@@ -1,14 +1,230 @@
import * as plugins from './plugins.js';
import type { IArchiveEntry, TCompressionLevel } from '../ts_shared/interfaces.js';
import { TarTools as SharedTarTools } from '../ts_shared/classes.tartools.js';
import { GzipTools } from '../ts_shared/classes.gziptools.js';
/**
* Extended TAR archive utilities with Node.js filesystem support
* Options for adding a file to a TAR pack stream
*/
export interface ITarPackFileOptions {
fileName: string;
content: string | Buffer | Uint8Array | plugins.stream.Readable;
size?: number;
mode?: number;
mtime?: Date;
}
/**
* Extended TAR archive utilities with Node.js streaming support
*
* For small archives: Use inherited buffer-based methods (packFiles, extractTar, etc.)
* For large archives: Use streaming methods (getPackStream, getExtractStream, etc.)
*/
export class TarTools extends SharedTarTools {
// ============================================
// STREAMING PACK METHODS (for large files)
// ============================================
/**
* Pack a directory into a TAR buffer (Node.js only)
* Get a streaming TAR pack instance
* Use this for packing large files without buffering everything in memory
*
* @example
* ```typescript
* const pack = tarTools.getPackStream();
*
* await tarTools.addFileToPack(pack, { fileName: 'large.bin', content: readStream, size: fileSize });
* await tarTools.addFileToPack(pack, { fileName: 'small.txt', content: 'Hello World' });
*
* pack.finalize();
* pack.pipe(fs.createWriteStream('output.tar'));
* ```
*/
public getPackStream(): plugins.tarStream.Pack {
return plugins.tarStream.pack();
}
/**
* Add a file to a TAR pack stream
* Supports strings, buffers, and readable streams
*
* @param pack - The pack stream from getPackStream()
* @param options - File options including name, content, and optional metadata
*/
public async addFileToPack(
pack: plugins.tarStream.Pack,
options: ITarPackFileOptions
): Promise<void> {
const { fileName, content, mode = 0o644, mtime = new Date() } = options;
if (typeof content === 'string') {
// String content - convert to buffer
const buffer = Buffer.from(content, 'utf8');
const entry = pack.entry({
name: fileName,
size: buffer.length,
mode,
mtime,
});
entry.write(buffer);
entry.end();
} else if (Buffer.isBuffer(content) || content instanceof Uint8Array) {
// Buffer content
const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content);
const entry = pack.entry({
name: fileName,
size: buffer.length,
mode,
mtime,
});
entry.write(buffer);
entry.end();
} else if (content && typeof (content as any).pipe === 'function') {
// Readable stream - requires size to be provided
const size = options.size;
if (size === undefined) {
throw new Error('Size must be provided when adding a stream to TAR pack');
}
return new Promise<void>((resolve, reject) => {
const entry = pack.entry({
name: fileName,
size,
mode,
mtime,
}, (err) => {
if (err) reject(err);
else resolve();
});
(content as plugins.stream.Readable).pipe(entry);
});
} else {
throw new Error('Unsupported content type for TAR entry');
}
}
// ============================================
// STREAMING EXTRACT METHODS (for large files)
// ============================================
/**
* Get a streaming TAR extract instance
* Use this for extracting large archives without buffering everything in memory
*
* @example
* ```typescript
* const extract = tarTools.getExtractStream();
*
* extract.on('entry', (header, stream, next) => {
* console.log(`Extracting: ${header.name}`);
* stream.pipe(fs.createWriteStream(`./out/${header.name}`));
* stream.on('end', next);
* });
*
* fs.createReadStream('archive.tar').pipe(extract);
* ```
*/
public getExtractStream(): plugins.tarStream.Extract {
return plugins.tarStream.extract();
}
/**
* Extract a TAR stream to a directory with true streaming (no buffering)
*
* @param sourceStream - The TAR archive stream
* @param targetDir - Directory to extract files to
*/
public async extractToDirectory(
sourceStream: plugins.stream.Readable,
targetDir: string
): Promise<void> {
await plugins.fsPromises.mkdir(targetDir, { recursive: true });
return new Promise<void>((resolve, reject) => {
const extract = this.getExtractStream();
extract.on('entry', async (header, stream, next) => {
const filePath = plugins.path.join(targetDir, header.name);
if (header.type === 'directory') {
await plugins.fsPromises.mkdir(filePath, { recursive: true });
stream.resume(); // Drain the stream
next();
} else if (header.type === 'file') {
await plugins.fsPromises.mkdir(plugins.path.dirname(filePath), { recursive: true });
const writeStream = plugins.fs.createWriteStream(filePath);
stream.pipe(writeStream);
writeStream.on('finish', next);
writeStream.on('error', reject);
} else {
stream.resume(); // Skip other types
next();
}
});
extract.on('finish', resolve);
extract.on('error', reject);
sourceStream.pipe(extract);
});
}
// ============================================
// STREAMING DIRECTORY PACK (for large directories)
// ============================================
/**
* Pack a directory into a TAR stream with true streaming (no buffering)
* Files are read and written one at a time, never loading everything into memory
*/
public async getDirectoryPackStream(directoryPath: string): Promise<plugins.tarStream.Pack> {
const pack = this.getPackStream();
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
// Process files sequentially to avoid memory issues
(async () => {
for (const filePath of fileTree) {
const absolutePath = plugins.path.join(directoryPath, filePath);
const stat = await plugins.fsPromises.stat(absolutePath);
if (stat.isFile()) {
const readStream = plugins.fs.createReadStream(absolutePath);
await this.addFileToPack(pack, {
fileName: filePath,
content: readStream,
size: stat.size,
mode: stat.mode,
mtime: stat.mtime,
});
}
}
pack.finalize();
})().catch((err) => pack.destroy(err));
return pack;
}
/**
* Pack a directory into a TAR.GZ stream with true streaming
* Uses Node.js zlib for streaming compression
*/
public async getDirectoryPackStreamGz(
directoryPath: string,
compressionLevel?: TCompressionLevel
): Promise<plugins.stream.Readable> {
const tarStream = await this.getDirectoryPackStream(directoryPath);
const { createGzip } = await import('node:zlib');
const gzip = createGzip({ level: compressionLevel ?? 6 });
return tarStream.pipe(gzip);
}
// ============================================
// BUFFER-BASED METHODS (inherited + filesystem)
// ============================================
/**
* Pack a directory into a TAR buffer (loads all files into memory)
* For large directories, use getDirectoryPackStream() instead
*/
public async packDirectory(directoryPath: string): Promise<Uint8Array> {
const fileTree = await plugins.listFileTree(directoryPath, '**/*');
@@ -16,36 +232,41 @@ export class TarTools extends SharedTarTools {
for (const filePath of fileTree) {
const absolutePath = plugins.path.join(directoryPath, filePath);
const content = await plugins.fsPromises.readFile(absolutePath);
entries.push({
archivePath: filePath,
content: new Uint8Array(content),
});
const stat = await plugins.fsPromises.stat(absolutePath);
if (stat.isFile()) {
const content = await plugins.fsPromises.readFile(absolutePath);
entries.push({
archivePath: filePath,
content: new Uint8Array(content),
});
}
}
return this.packFiles(entries);
}
/**
* Pack a directory into a TAR.GZ buffer (Node.js only)
* Pack a directory into a TAR.GZ buffer (loads all files into memory)
* For large directories, use getDirectoryPackStreamGz() instead
*/
public async packDirectoryToTarGz(
directoryPath: string,
compressionLevel?: TCompressionLevel
): Promise<Uint8Array> {
const tarBuffer = await this.packDirectory(directoryPath);
const gzipTools = new GzipTools();
return gzipTools.compress(tarBuffer, compressionLevel);
const { gzipSync } = await import('fflate');
return gzipSync(new Uint8Array(tarBuffer), { level: compressionLevel ?? 6 });
}
/**
* Pack a directory into a TAR.GZ stream (Node.js only)
* Pack a directory into a TAR.GZ stream
* This is now a true streaming implementation
*/
public async packDirectoryToTarGzStream(
directoryPath: string,
compressionLevel?: TCompressionLevel
): Promise<plugins.stream.Readable> {
const buffer = await this.packDirectoryToTarGz(directoryPath, compressionLevel);
return plugins.stream.Readable.from(buffer);
return this.getDirectoryPackStreamGz(directoryPath, compressionLevel);
}
}

4
ts/index.ts
View File

@@ -7,5 +7,5 @@ export * from './classes.smartarchive.js';
// Node.js-specific: Archive analysis with SmartArchive integration
export * from './classes.archiveanalyzer.js';
// Node.js-specific: Extended TarTools with filesystem support (overrides shared TarTools)
export { TarTools } from './classes.tartools.js';
// Node.js-specific: Extended TarTools with streaming support (overrides shared TarTools)
export { TarTools, type ITarPackFileOptions } from './classes.tartools.js';

4
ts/plugins.ts
View File

@@ -47,3 +47,7 @@ export {
smartrx,
smarturl,
};
// Node.js-specific: tar-stream for true streaming TAR support
import * as tarStream from 'tar-stream';
export { tarStream };

2
ts_web/00_commitinfo_data.ts
View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartarchive',
version: '5.1.0',
version: '5.2.0',
description: 'A library for working with archive files, providing utilities for compressing and decompressing data.'
}