push.rocks/smartarchive
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

fix(ziptools,gziptools): Use fflate synchronous APIs for ZIP and GZIP operations for Deno compatibility; add TEntryFilter type and small docs/tests cleanup

Browse Source
This commit is contained in:
Jürgen Kunz
2025-11-25 13:37:27 +00:00
parent 5cc030d433
commit 11bbddc763
12 changed files with 1300 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
test/test.gzip.node+deno.ts
View File

@@ -22,7 +22,7 @@ tap.test('should create and extract a gzip file', async () => {
const testContent = 'This is a test file for gzip compression and decompression.\n'.repeat(100);
const testFileName = 'test-file.txt';
const gzipFileName = 'test-file.txt.gz';
// Write the original file
await plugins.fsPromises.writeFile(
plugins.path.join(testPaths.gzipTestDir, testFileName),
@@ -36,24 +36,22 @@ tap.test('should create and extract a gzip file', async () => {
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
Buffer.from(compressed)
);
// Now test extraction using SmartArchive
const gzipArchive = await smartarchive.SmartArchive.fromFile(
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
);
// Export to a new location
// Now test extraction using SmartArchive fluent API
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'extracted');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
// Provide a filename since gzip doesn't contain filename metadata
await gzipArchive.extractToDirectory(extractPath, { fileName: 'test-file.txt' });
await smartarchive.SmartArchive.create()
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
.fileName('test-file.txt')
.extract(extractPath);
// Read the extracted file
const extractedContent = await plugins.fsPromises.readFile(
plugins.path.join(extractPath, 'test-file.txt'),
'utf8'
);
// Verify the content matches
expect(extractedContent).toEqual(testContent);
});
@@ -62,7 +60,7 @@ tap.test('should handle gzip stream extraction', async () => {
// Create test data
const testContent = 'Stream test data for gzip\n'.repeat(50);
const gzipFileName = 'stream-test.txt.gz';
// Create gzip compressed version
const fflate = await import('fflate');
const compressed = fflate.gzipSync(Buffer.from(testContent));
@@ -75,14 +73,13 @@ tap.test('should handle gzip stream extraction', async () => {
const gzipStream = plugins.fs.createReadStream(
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
);
// Test extraction using SmartArchive from stream
const gzipArchive = await smartarchive.SmartArchive.fromStream(gzipStream);
// Export to stream and collect the result
// Test extraction using SmartArchive from stream with fluent API
const streamFiles: any[] = [];
const resultStream = await gzipArchive.extractToStream();
const resultStream = await smartarchive.SmartArchive.create()
.stream(gzipStream)
.toStreamFiles();
await new Promise<void>((resolve, reject) => {
resultStream.on('data', (streamFile) => {
streamFiles.push(streamFile);
@@ -90,10 +87,10 @@ tap.test('should handle gzip stream extraction', async () => {
resultStream.on('end', resolve);
resultStream.on('error', reject);
});
// Verify we got the expected file
expect(streamFiles.length).toBeGreaterThan(0);
// Read content from the stream file
if (streamFiles[0]) {
const chunks: Buffer[] = [];
@@ -103,7 +100,7 @@ tap.test('should handle gzip stream extraction', async () => {
readStream.on('end', resolve);
readStream.on('error', reject);
});
const extractedContent = Buffer.concat(chunks).toString();
expect(extractedContent).toEqual(testContent);
}
@@ -112,36 +109,32 @@ tap.test('should handle gzip stream extraction', async () => {
tap.test('should handle gzip files with original filename in header', async () => {
// Test with a real-world gzip file that includes filename in header
const testContent = 'File with name in gzip header\n'.repeat(30);
const originalFileName = 'original-name.log';
const gzipFileName = 'compressed.gz';
// Create a proper gzip with filename header using Node's zlib
const zlib = await import('node:zlib');
const gzipBuffer = await new Promise<Buffer>((resolve, reject) => {
zlib.gzip(Buffer.from(testContent), {
zlib.gzip(Buffer.from(testContent), {
level: 9,
// Note: Node's zlib doesn't support embedding filename directly,
// but we can test the extraction anyway
}, (err, result) => {
if (err) reject(err);
else resolve(result);
});
});
await plugins.fsPromises.writeFile(
plugins.path.join(testPaths.gzipTestDir, gzipFileName),
gzipBuffer
);
// Test extraction
const gzipArchive = await smartarchive.SmartArchive.fromFile(
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
);
// Test extraction with fluent API
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'header-test');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
// Provide a filename since gzip doesn't reliably contain filename metadata
await gzipArchive.extractToDirectory(extractPath, { fileName: 'compressed.txt' });
await smartarchive.SmartArchive.create()
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
.fileName('compressed.txt')
.extract(extractPath);
// Check if file was extracted (name might be derived from archive name)
const files = await plugins.listFileTree(extractPath, '**/*');
@@ -160,7 +153,7 @@ tap.test('should handle large gzip files', async () => {
// Create a larger test file
const largeContent = 'x'.repeat(1024 * 1024); // 1MB of 'x' characters
const gzipFileName = 'large-file.txt.gz';
// Compress the large file
const fflate = await import('fflate');
const compressed = fflate.gzipSync(Buffer.from(largeContent));
@@ -169,15 +162,14 @@ tap.test('should handle large gzip files', async () => {
Buffer.from(compressed)
);
// Test extraction
const gzipArchive = await smartarchive.SmartArchive.fromFile(
plugins.path.join(testPaths.gzipTestDir, gzipFileName)
);
// Test extraction with fluent API
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'large-extracted');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
// Provide a filename since gzip doesn't contain filename metadata
await gzipArchive.extractToDirectory(extractPath, { fileName: 'large-file.txt' });
await smartarchive.SmartArchive.create()
.file(plugins.path.join(testPaths.gzipTestDir, gzipFileName))
.fileName('large-file.txt')
.extract(extractPath);
// Verify the extracted content
const files = await plugins.listFileTree(extractPath, '**/*');
@@ -195,14 +187,13 @@ tap.test('should handle real-world multi-chunk gzip from URL', async () => {
// Test with a real tgz file that will be processed in multiple chunks
const testUrl = 'https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz';
// Download and extract the archive
const testArchive = await smartarchive.SmartArchive.fromUrl(testUrl);
// Download and extract the archive with fluent API
const extractPath = plugins.path.join(testPaths.gzipTestDir, 'real-world-test');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
// This will test multi-chunk decompression as the file is larger
await testArchive.extractToDirectory(extractPath);
await smartarchive.SmartArchive.create()
.url(testUrl)
.extract(extractPath);
// Verify extraction worked
const files = await plugins.listFileTree(extractPath, '**/*');
@@ -265,22 +256,17 @@ tap.test('should handle real-world multi-chunk gzip from URL', async () => {
tap.test('should handle gzip extraction fully in memory', async () => {
// Create test data in memory
const testContent = 'This is test data for in-memory gzip processing\n'.repeat(100);
// Compress using fflate in memory
const fflate = await import('fflate');
const compressed = fflate.gzipSync(Buffer.from(testContent));
// Create a stream from the compressed data
const { Readable } = await import('node:stream');
const compressedStream = Readable.from(Buffer.from(compressed));
// Process through SmartArchive without touching filesystem
const gzipArchive = await smartarchive.SmartArchive.fromStream(compressedStream);
// Export to stream of stream files (in memory)
// Process through SmartArchive without touching filesystem using fluent API
const streamFiles: plugins.smartfile.StreamFile[] = [];
const resultStream = await gzipArchive.extractToStream();
const resultStream = await smartarchive.SmartArchive.create()
.buffer(Buffer.from(compressed))
.toStreamFiles();
await new Promise<void>((resolve, reject) => {
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
streamFiles.push(streamFile);
@@ -288,21 +274,21 @@ tap.test('should handle gzip extraction fully in memory', async () => {
resultStream.on('end', resolve);
resultStream.on('error', reject);
});
// Verify we got a file
expect(streamFiles.length).toBeGreaterThan(0);
// Read the content from memory without filesystem
const firstFile = streamFiles[0];
const chunks: Buffer[] = [];
const readStream = await firstFile.createReadStream();
await new Promise<void>((resolve, reject) => {
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
readStream.on('end', resolve);
readStream.on('error', reject);
});
const extractedContent = Buffer.concat(chunks).toString();
expect(extractedContent).toEqual(testContent);
console.log(` ✓ In-memory extraction successful (${extractedContent.length} bytes)`);
@@ -314,88 +300,83 @@ tap.test('should handle real tgz file fully in memory', async (tools) => {
const response = await plugins.smartrequest.SmartRequest.create()
.url('https://registry.npmjs.org/@push.rocks/smartfile/-/smartfile-11.2.7.tgz')
.get();
const tgzBuffer = Buffer.from(await response.arrayBuffer());
console.log(` Downloaded ${tgzBuffer.length} bytes into memory`);
// Create stream from buffer
const { Readable: Readable2 } = await import('node:stream');
const tgzStream = Readable2.from(tgzBuffer);
// Process through SmartArchive in memory
const archive = await smartarchive.SmartArchive.fromStream(tgzStream);
// Export to stream of stream files (in memory)
// Process through SmartArchive in memory with fluent API
const streamFiles: plugins.smartfile.StreamFile[] = [];
const resultStream = await archive.extractToStream();
const resultStream = await smartarchive.SmartArchive.create()
.buffer(tgzBuffer)
.toStreamFiles();
await new Promise<void>((resolve, reject) => {
let timeout: NodeJS.Timeout;
const cleanup = () => {
clearTimeout(timeout);
};
timeout = setTimeout(() => {
cleanup();
resolve(); // Resolve after timeout if stream doesn't end
}, 5000);
resultStream.on('data', (streamFile: plugins.smartfile.StreamFile) => {
streamFiles.push(streamFile);
});
resultStream.on('end', () => {
cleanup();
resolve();
});
resultStream.on('error', (err) => {
cleanup();
reject(err);
});
});
console.log(` Extracted ${streamFiles.length} files in memory`);
// At minimum we should have extracted something
expect(streamFiles.length).toBeGreaterThan(0);
// Find and read package.json from memory
const packageJsonFile = streamFiles.find(f => f.relativeFilePath?.includes('package.json'));
if (packageJsonFile) {
const chunks: Buffer[] = [];
const readStream = await packageJsonFile.createReadStream();
await new Promise<void>((resolve, reject) => {
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
readStream.on('end', resolve);
readStream.on('error', reject);
});
const packageJsonContent = Buffer.concat(chunks).toString();
const packageJson = JSON.parse(packageJsonContent);
expect(packageJson.name).toEqual('@push.rocks/smartfile');
expect(packageJson.version).toEqual('11.2.7');
console.log(` ✓ Read package.json from memory: ${packageJson.name}@${packageJson.version}`);
}
// Read a few more files to verify integrity
const filesToCheck = streamFiles.slice(0, 3);
for (const file of filesToCheck) {
const chunks: Buffer[] = [];
const readStream = await file.createReadStream();
await new Promise<void>((resolve, reject) => {
readStream.on('data', (chunk: Buffer) => chunks.push(chunk));
readStream.on('end', resolve);
readStream.on('error', reject);
});
const content = Buffer.concat(chunks);
expect(content.length).toBeGreaterThan(0);
console.log(` ✓ Read ${file.relativeFilePath} from memory (${content.length} bytes)`);
}
});
export default tap.start();
export default tap.start();

209
test/test.node+deno.ts
View File

@@ -32,20 +32,209 @@ tap.preTask('should prepare downloads', async (tools) => {
);
});
tap.test('should extract existing files on disk', async () => {
const testSmartarchive = await smartarchive.SmartArchive.fromUrl(
'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz',
);
await testSmartarchive.extractToDirectory(testPaths.nogitDir);
tap.test('should extract existing files on disk using fluent API', async () => {
await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.extract(testPaths.nogitDir);
});
tap.test('should extract from file using fluent API', async () => {
const extractPath = plugins.path.join(testPaths.nogitDir, 'from-file-test');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
await smartarchive.SmartArchive.create()
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
.extract(extractPath);
const files = await plugins.listFileTree(extractPath, '**/*');
expect(files.length).toBeGreaterThan(0);
});
tap.test('should extract with stripComponents using fluent API', async () => {
const extractPath = plugins.path.join(testPaths.nogitDir, 'strip-test');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.stripComponents(1)
.extract(extractPath);
const files = await plugins.listFileTree(extractPath, '**/*');
expect(files.length).toBeGreaterThan(0);
// Files should not have 'package/' prefix
const hasPackagePrefix = files.some(f => f.startsWith('package/'));
expect(hasPackagePrefix).toBeFalse();
});
tap.test('should extract with filter using fluent API', async () => {
const extractPath = plugins.path.join(testPaths.nogitDir, 'filter-test');
await plugins.fsPromises.mkdir(extractPath, { recursive: true });
await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.filter(entry => entry.path.endsWith('.json'))
.extract(extractPath);
const files = await plugins.listFileTree(extractPath, '**/*');
// All extracted files should be JSON
for (const file of files) {
expect(file.endsWith('.json')).toBeTrue();
}
});
tap.test('should list archive entries using fluent API', async () => {
const entries = await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.list();
expect(entries.length).toBeGreaterThan(0);
const hasPackageJson = entries.some(e => e.path.includes('package.json'));
expect(hasPackageJson).toBeTrue();
});
tap.test('should create archive using fluent API', async () => {
const archive = await smartarchive.SmartArchive.create()
.format('tar.gz')
.compression(9)
.entry('hello.txt', 'Hello World!')
.entry('config.json', JSON.stringify({ name: 'test', version: '1.0.0' }));
expect(archive).toBeInstanceOf(smartarchive.SmartArchive);
const buffer = await archive.toBuffer();
expect(buffer.length).toBeGreaterThan(0);
});
tap.test('should create and write archive to file using fluent API', async () => {
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.tar.gz');
await smartarchive.SmartArchive.create()
.format('tar.gz')
.entry('readme.txt', 'This is a test archive')
.entry('data/info.json', JSON.stringify({ created: new Date().toISOString() }))
.toFile(outputPath);
// Verify file was created
const stats = await plugins.fsPromises.stat(outputPath);
expect(stats.size).toBeGreaterThan(0);
// Verify we can extract it
const extractPath = plugins.path.join(testPaths.nogitDir, 'verify-created');
await smartarchive.SmartArchive.create()
.file(outputPath)
.extract(extractPath);
const files = await plugins.listFileTree(extractPath, '**/*');
expect(files).toContain('readme.txt');
});
tap.test('should create ZIP archive using fluent API', async () => {
const outputPath = plugins.path.join(testPaths.nogitDir, 'created-archive.zip');
await smartarchive.SmartArchive.create()
.format('zip')
.entry('file1.txt', 'Content 1')
.entry('file2.txt', 'Content 2')
.toFile(outputPath);
// Verify file was created
const stats = await plugins.fsPromises.stat(outputPath);
expect(stats.size).toBeGreaterThan(0);
});
tap.test('should extract to SmartFiles using fluent API', async () => {
const smartFiles = await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.toSmartFiles();
expect(smartFiles.length).toBeGreaterThan(0);
const packageJson = smartFiles.find(f => f.relative.includes('package.json'));
expect(packageJson).toBeDefined();
});
tap.test('should analyze archive using fluent API', async () => {
const info = await smartarchive.SmartArchive.create()
.file(plugins.path.join(testPaths.nogitDir, 'test.tgz'))
.analyze();
expect(info.isArchive).toBeTrue();
expect(info.isCompressed).toBeTrue();
expect(info.format).toEqual('gz');
});
tap.test('should check if file exists in archive using fluent API', async () => {
const hasPackageJson = await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.hasFile('package.json');
expect(hasPackageJson).toBeTrue();
});
tap.test('should extract single file using fluent API', async () => {
const packageJson = await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.extractFile('package.json');
expect(packageJson).toBeDefined();
expect(packageJson!.contents.toString()).toContain('websetup');
});
tap.test('should handle include/exclude patterns', async () => {
const smartFiles = await smartarchive.SmartArchive.create()
.url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz')
.include(/\.json$/)
.toSmartFiles();
expect(smartFiles.length).toBeGreaterThan(0);
for (const file of smartFiles) {
expect(file.relative.endsWith('.json')).toBeTrue();
}
});
tap.test('should throw error when mixing modes', async () => {
let threw = false;
try {
smartarchive.SmartArchive.create()
.url('https://example.com/archive.tgz')
.entry('file.txt', 'content'); // This should throw
} catch (e) {
threw = true;
expect((e as Error).message).toContain('extraction mode');
}
expect(threw).toBeTrue();
});
tap.test('should throw error when no source configured', async () => {
let threw = false;
try {
await smartarchive.SmartArchive.create().extract('./output');
} catch (e) {
threw = true;
expect((e as Error).message).toContain('No source configured');
}
expect(threw).toBeTrue();
});
tap.test('should throw error when no format configured', async () => {
let threw = false;
try {
await smartarchive.SmartArchive.create()
.entry('file.txt', 'content')
.toBuffer();
} catch (e) {
threw = true;
expect((e as Error).message).toContain('No format specified');
}
expect(threw).toBeTrue();
});
tap.skip.test('should extract a b2zip', async () => {
const dataUrl =
'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
const testArchive = await smartarchive.SmartArchive.fromUrl(dataUrl);
await testArchive.extractToDirectory(
plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'),
);
await smartarchive.SmartArchive.create()
.url(dataUrl)
.extract(plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'));
});
await tap.start();
export default tap.start();