From da22f375d21ae4daea64abee0c595c0929bcea29 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Sat, 11 Nov 2023 18:28:50 +0100 Subject: [PATCH] fix(core): update --- license | 6 +- package.json | 3 +- pnpm-lock.yaml | 23 +-- ts/00_commitinfo_data.ts | 2 +- ts/bzip2/bititerator.ts | 41 +++++ ts/bzip2/bzip2.ts | 335 ++++++++++++++++++++++++++++++++++ ts/bzip2/index.ts | 84 +++++++++ ts/classes.archiveanalyzer.ts | 8 +- ts/classes.bzip2tools.ts | 33 +--- ts/classes.smartarchive.ts | 6 +- ts/plugins.ts | 3 +- 11 files changed, 484 insertions(+), 60 deletions(-) create mode 100644 ts/bzip2/bititerator.ts create mode 100644 ts/bzip2/bzip2.ts create mode 100644 ts/bzip2/index.ts diff --git a/license b/license index eb3beb5..35295ec 100644 --- a/license +++ b/license @@ -1,6 +1,10 @@ The MIT License (MIT) -Copyright (c) 2016 Lossless GmbH +Copyright (c) 2016 Task Venture Capital GmbH + +Includes work under MIT license with copyrights: +* Copyright (c) 2017 by Jan Boelsche (jan@lagomorph.de) +* Copyright 2011 by antimatter15 (antimatter15@gmail.com) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/package.json b/package.json index cb029b4..13652d6 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,10 @@ "@push.rocks/smartunique": "^3.0.6", "@push.rocks/smarturl": "^3.0.7", "@types/tar-stream": "^3.1.3", - "@types/unbzip2-stream": "^1.4.3", "fflate": "^0.8.1", "file-type": "^18.6.0", "tar-stream": "^3.1.6", - "unbzip2-stream": "^1.4.3" + "through": "^2.3.8" }, "devDependencies": { "@git.zone/tsbuild": "^2.1.66", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8f3f1cc..238d6d8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,9 +32,6 @@ dependencies: '@types/tar-stream': specifier: ^3.1.3 version: 3.1.3 - '@types/unbzip2-stream': - specifier: ^1.4.3 - version: 1.4.3 fflate: specifier: ^0.8.1 version: 0.8.1 @@ -44,9 +41,9 @@ dependencies: tar-stream: specifier: ^3.1.6 version: 3.1.6 - unbzip2-stream: - specifier: ^1.4.3 - version: 1.4.3 + through: + specifier: ^2.3.8 + version: 2.3.8 devDependencies: '@git.zone/tsbuild': @@ -1671,12 +1668,6 @@ packages: dependencies: '@types/node': 20.8.10 - /@types/through@0.0.33: - resolution: {integrity: sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==} - dependencies: - '@types/node': 20.9.0 - dev: false - /@types/trusted-types@2.0.5: resolution: {integrity: sha512-I3pkr8j/6tmQtKV/ZzHtuaqYSQvyjGRKH4go60Rr0IDLlFxuRT5V32uvB1mecM5G1EVAUyF/4r4QZ1GHgz+mxA==} dev: true @@ -1687,12 +1678,6 @@ packages: source-map: 0.6.1 dev: true - /@types/unbzip2-stream@1.4.3: - resolution: {integrity: sha512-D8X5uuJRISqc8YtwL8jNW2FpPdUOCYXbfD6zNROCTbVXK9nawucxh10tVXE3MPjnHdRA1LvB0zDxVya/lBsnYw==} - dependencies: - '@types/through': 0.0.33 - dev: false - /@types/uuid@7.0.7: resolution: {integrity: sha512-xlbA6IkLhAK9G0hn6qli0ygl8KOKhONfuEvfoBrs9QX++g6v2qF//xGTYx+AKs5e151s2V1J7qWuABqaT+lGXw==} dev: true @@ -2103,6 +2088,7 @@ packages: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + dev: true /buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} @@ -5368,6 +5354,7 @@ packages: dependencies: buffer: 5.7.1 through: 2.3.8 + dev: true /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 1e86f4a..ce2c9cc 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartarchive', - version: '4.0.6', + version: '4.0.7', description: 'work with archives' } diff --git a/ts/bzip2/bititerator.ts b/ts/bzip2/bititerator.ts new file mode 100644 index 0000000..3c6b335 --- /dev/null +++ b/ts/bzip2/bititerator.ts @@ -0,0 +1,41 @@ +var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF]; + +// returns a function that reads bits. +// takes a buffer iterator as input +export function bitIterator(nextBuffer: () => Buffer) { + var bit = 0, byte = 0; + var bytes = nextBuffer(); + var f = function(n) { + if (n === null && bit != 0) { // align to byte boundary + bit = 0 + byte++; + return; + } + var result = 0; + while(n > 0) { + if (byte >= bytes.length) { + byte = 0; + bytes = nextBuffer(); + } + var left = 8 - bit; + if (bit === 0 && n > 0) + // @ts-ignore + f.bytesRead++; + if (n >= left) { + result <<= left; + result |= (BITMASK[left] & bytes[byte++]); + bit = 0; + n -= left; + } else { + result <<= n; + result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); + bit += n; + n = 0; + } + } + return result; + }; + // @ts-ignore + f.bytesRead = 0; + return f; +}; \ No newline at end of file diff --git a/ts/bzip2/bzip2.ts b/ts/bzip2/bzip2.ts new file mode 100644 index 0000000..245ef23 --- /dev/null +++ b/ts/bzip2/bzip2.ts @@ -0,0 +1,335 @@ +export class Bzip2Error extends Error { + public name: string = 'Bzip2Error'; + public message: string; + public stack = (new Error()).stack; + + constructor(messageArg: string) { + super(); + this.message = messageArg; + } +} + +var messageArg = { + Error: function(message) {throw new Bzip2Error(message);} +}; + +export class Bzip2 { + public Bzip2Error = Bzip2Error; + public crcTable = + [ + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, + 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, + 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, + 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, + 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, + 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, + 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, + 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, + 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, + 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, + 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, + 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, + 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, + 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, + 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, + 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, + 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, + 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, + 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, + 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, + 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, + 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + ]; + + array = function(bytes) { + var bit = 0, byte = 0; + var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; + return function(n) { + var result = 0; + while(n > 0) { + var left = 8 - bit; + if (n >= left) { + result <<= left; + result |= (BITMASK[left] & bytes[byte++]); + bit = 0; + n -= left; + } else { + result <<= n; + result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); + bit += n; + n = 0; + } + } + return result; + } + } + + simple = function(srcbuffer, stream) { + var bits = this.array(srcbuffer); + var size = this.header(bits); + var ret = false; + var bufsize = 100000 * size; + var buf = new Int32Array(bufsize); + + do { + ret = this.decompress(bits, stream, buf, bufsize); + } while(!ret); + } + + header = function(bits) { + this.byteCount = new Int32Array(256); + this.symToByte = new Uint8Array(256); + this.mtfSymbol = new Int32Array(256); + this.selectors = new Uint8Array(0x8000); + + if (bits(8*3) != 4348520) messageArg.Error("No magic number found"); + + var i = bits(8) - 48; + if (i < 1 || i > 9) messageArg.Error("Not a BZIP archive"); + return i; + }; + + decompress = function(bits, stream, buf, bufsize, streamCRC) { + var MAX_HUFCODE_BITS = 20; + var MAX_SYMBOLS = 258; + var SYMBOL_RUNA = 0; + var SYMBOL_RUNB = 1; + var GROUP_SIZE = 50; + var crc = 0 ^ (-1); + + for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); + if (h == "177245385090") { + var finalCRC = bits(32)|0; + if (finalCRC !== streamCRC) messageArg.Error("Error in bzip2: crc32 do not match"); + // align stream to byte + bits(null); + return null; // reset streamCRC for next call + } + if (h != "314159265359") messageArg.Error("eek not valid bzip data"); + var crcblock = bits(32)|0; // CRC code + if (bits(1)) messageArg.Error("unsupported obsolete version"); + var origPtr = bits(24); + if (origPtr > bufsize) messageArg.Error("Initial position larger than buffer size"); + var t = bits(16); + var symTotal = 0; + for (i = 0; i < 16; i++) { + if (t & (1 << (15 - i))) { + var k = bits(16); + for(j = 0; j < 16; j++) { + if (k & (1 << (15 - j))) { + this.symToByte[symTotal++] = (16 * i) + j; + } + } + } + } + + var groupCount = bits(3); + if (groupCount < 2 || groupCount > 6) messageArg.Error("another error"); + var nSelectors = bits(15); + if (nSelectors == 0) messageArg.Error("meh"); + for(var i = 0; i < groupCount; i++) this.mtfSymbol[i] = i; + + for(var i = 0; i < nSelectors; i++) { + for(var j = 0; bits(1); j++) if (j >= groupCount) messageArg.Error("whoops another error"); + var uc = this.mtfSymbol[j]; + for(var k: any = j-1; k>=0; k--) { + this.mtfSymbol[k+1] = this.mtfSymbol[k]; + } + this.mtfSymbol[0] = uc; + this.selectors[i] = uc; + } + + var symCount = symTotal + 2; + var groups = []; + var length = new Uint8Array(MAX_SYMBOLS), + temp = new Uint16Array(MAX_HUFCODE_BITS+1); + + var hufGroup; + + for(var j = 0; j < groupCount; j++) { + t = bits(5); //lengths + for(var i = 0; i < symCount; i++) { + while(true){ + if (t < 1 || t > MAX_HUFCODE_BITS) messageArg.Error("I gave up a while ago on writing error messages"); + if (!bits(1)) break; + if (!bits(1)) t++; + else t--; + } + length[i] = t; + } + var minLen, maxLen; + minLen = maxLen = length[0]; + for(var i = 1; i < symCount; i++) { + if (length[i] > maxLen) maxLen = length[i]; + else if (length[i] < minLen) minLen = length[i]; + } + hufGroup = groups[j] = {}; + hufGroup.permute = new Int32Array(MAX_SYMBOLS); + hufGroup.limit = new Int32Array(MAX_HUFCODE_BITS + 1); + hufGroup.base = new Int32Array(MAX_HUFCODE_BITS + 1); + + hufGroup.minLen = minLen; + hufGroup.maxLen = maxLen; + var base = hufGroup.base; + var limit = hufGroup.limit; + var pp = 0; + for(var i: number = minLen; i <= maxLen; i++) + for(var t: any = 0; t < symCount; t++) + if (length[t] == i) hufGroup.permute[pp++] = t; + for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; + for(i = 0; i < symCount; i++) temp[length[i]]++; + pp = t = 0; + for(i = minLen; i < maxLen; i++) { + pp += temp[i]; + limit[i] = pp - 1; + pp <<= 1; + base[i+1] = pp - (t += temp[i]); + } + limit[maxLen] = pp + temp[maxLen] - 1; + base[minLen] = 0; + } + + for(var i = 0; i < 256; i++) { + this.mtfSymbol[i] = i; + this.byteCount[i] = 0; + } + var runPos, count, symCount: number, selector; + runPos = count = symCount = selector = 0; + while(true) { + if (!(symCount--)) { + symCount = GROUP_SIZE - 1; + if (selector >= nSelectors) messageArg.Error("meow i'm a kitty, that's an error"); + hufGroup = groups[this.selectors[selector++]]; + base = hufGroup.base; + limit = hufGroup.limit; + } + i = hufGroup.minLen; + j = bits(i); + while(true) { + if (i > hufGroup.maxLen) messageArg.Error("rawr i'm a dinosaur"); + if (j <= limit[i]) break; + i++; + j = (j << 1) | bits(1); + } + j -= base[i]; + if (j < 0 || j >= MAX_SYMBOLS) messageArg.Error("moo i'm a cow"); + var nextSym = hufGroup.permute[j]; + if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { + if (!runPos){ + runPos = 1; + t = 0; + } + if (nextSym == SYMBOL_RUNA) t += runPos; + else t += 2 * runPos; + runPos <<= 1; + continue; + } + if (runPos) { + runPos = 0; + if (count + t > bufsize) messageArg.Error("Boom."); + uc = this.symToByte[this.mtfSymbol[0]]; + this.byteCount[uc] += t; + while(t--) buf[count++] = uc; + } + if (nextSym > symTotal) break; + if (count >= bufsize) messageArg.Error("I can't think of anything. Error"); + i = nextSym - 1; + uc = this.mtfSymbol[i]; + for(var k: any = i-1; k>=0; k--) { + this.mtfSymbol[k+1] = this.mtfSymbol[k]; + } + this.mtfSymbol[0] = uc + uc = this.symToByte[uc]; + this.byteCount[uc]++; + buf[count++] = uc; + } + if (origPtr < 0 || origPtr >= count) messageArg.Error("I'm a monkey and I'm throwing something at someone, namely you"); + var j = 0; + for(var i = 0; i < 256; i++) { + k = j + this.byteCount[i]; + this.byteCount[i] = j; + j = k; + } + for(var i = 0; i < count; i++) { + uc = buf[i] & 0xff; + buf[this.byteCount[uc]] |= (i << 8); + this.byteCount[uc]++; + } + var pos = 0, current = 0, run = 0; + if (count) { + pos = buf[origPtr]; + current = (pos & 0xff); + pos >>= 8; + run = -1; + } + count = count; + var copies, previous, outbyte; + while(count) { + count--; + previous = current; + pos = buf[pos]; + current = pos & 0xff; + pos >>= 8; + if (run++ == 3) { + copies = current; + outbyte = previous; + current = -1; + } else { + copies = 1; + outbyte = current; + } + while(copies--) { + crc = ((crc << 8) ^ this.crcTable[((crc>>24) ^ outbyte) & 0xFF])&0xFFFFFFFF; // crc32 + stream(outbyte); + } + if (current != previous) run = 0; + } + + crc = (crc ^ (-1)) >>> 0; + if ((crc|0) != (crcblock|0)) messageArg.Error("Error in bzip2: crc32 do not match"); + streamCRC = (crc ^ ((streamCRC << 1) | (streamCRC >>> 31))) & 0xFFFFFFFF; + return streamCRC; + }; +}; \ No newline at end of file diff --git a/ts/bzip2/index.ts b/ts/bzip2/index.ts new file mode 100644 index 0000000..4c398a0 --- /dev/null +++ b/ts/bzip2/index.ts @@ -0,0 +1,84 @@ +import * as plugins from '../plugins.js'; + +import { Bzip2 } from './bzip2.js'; +import { bitIterator } from './bititerator.js'; + +export function unbzip2Stream() { + const bzip2Instance = new Bzip2(); + var bufferQueue = []; + var hasBytes = 0; + var blockSize = 0; + var broken = false; + var done = false; + var bitReader = null; + var streamCRC = null; + + function decompressBlock() { + if (!blockSize) { + blockSize = bzip2Instance.header(bitReader); + streamCRC = 0; + } else { + var bufsize = 100000 * blockSize; + var buf = new Int32Array(bufsize); + + var chunk = []; + var f = function (b) { + chunk.push(b); + }; + + streamCRC = bzip2Instance.decompress(bitReader, f, buf, bufsize, streamCRC); + if (streamCRC === null) { + // reset for next bzip2 header + blockSize = 0; + return; + } else { + return Buffer.from(chunk); + } + } + } + + var outlength = 0; + const decompressAndPush = async (smartDuplexStream: plugins.smartstream.SmartDuplex) => { + if (broken) return; + try { + const resultChunk = decompressBlock(); + smartDuplexStream.push(resultChunk); + if (resultChunk) { + //console.error('write at', outlength.toString(16)); + outlength += resultChunk.length; + } + } catch (e) { + console.error(e); + smartDuplexStream.emit('error', e); + broken = true; + return false; + } + }; + + return new plugins.smartstream.SmartDuplex({ + writeFunction: async function (data, streamTools) { + //console.error('received', data.length,'bytes in', typeof data); + bufferQueue.push(data); + hasBytes += data.length; + if (bitReader === null) { + bitReader = bitIterator(function () { + return bufferQueue.shift(); + }); + } + while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) { + //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer'); + await decompressAndPush(this); + } + }, + finalFunction: async function (streamTools) { + //console.error(x,'last compressing with', hasBytes, 'bytes in buffer'); + while (!broken && bitReader && hasBytes > bitReader.bytesRead) { + await decompressAndPush(this); + } + if (!broken) { + if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely')); + this.queue(null); + } + }, + }); +} diff --git a/ts/classes.archiveanalyzer.ts b/ts/classes.archiveanalyzer.ts index 01a42ef..158f678 100644 --- a/ts/classes.archiveanalyzer.ts +++ b/ts/classes.archiveanalyzer.ts @@ -30,14 +30,14 @@ export class ArchiveAnalyzer { } - private getDecompressionStream( + private async getDecompressionStream( mimeTypeArg: plugins.fileType.FileTypeResult['mime'] - ): plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract { + ): Promise { switch (mimeTypeArg) { case 'application/gzip': return this.smartArchiveRef.gzipTools.getDecompressionStream(); case 'application/x-bzip2': - return this.smartArchiveRef.bzip2Tools.getDecompressionStream(); // replace with your own bzip2 decompression stream + return await this.smartArchiveRef.bzip2Tools.getDecompressionStream(); // replace with your own bzip2 decompression stream case 'application/x-tar': return this.smartArchiveRef.tarTools.getDecompressionStream(); // replace with your own tar decompression stream default: @@ -53,7 +53,7 @@ export class ArchiveAnalyzer { readableObjectMode: true, writeFunction: async (chunkArg: Buffer, streamtools) => { const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg); - const decompressionStream = this.getDecompressionStream(fileType?.mime as any); + const decompressionStream = await this.getDecompressionStream(fileType?.mime as any); resultStream.push(chunkArg); if (firstRun) { firstRun = false; diff --git a/ts/classes.bzip2tools.ts b/ts/classes.bzip2tools.ts index ee7d067..d8e6415 100644 --- a/ts/classes.bzip2tools.ts +++ b/ts/classes.bzip2tools.ts @@ -1,36 +1,7 @@ import type { SmartArchive } from './classes.smartarchive.js'; import * as plugins from './plugins.js'; -export class DecompressBzip2Transform extends plugins.stream.Transform { - private bzip2Decompressor: plugins.stream.Transform; - - constructor() { - super(); - // Initialize the bzip2 decompressor once here - this.bzip2Decompressor = plugins.unbzip2Stream(); - this.bzip2Decompressor.on('data', (data: Buffer) => { - // When data is decompressed, push it to the stream - this.push(data); - }); - this.bzip2Decompressor.on('error', (err) => { - // If an error occurs, emit it on this stream - this.emit('error', err); - }); - } - - _transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) { - // Pass the chunk directly to the decompressor - // The decompressor will handle the state across chunks - this.bzip2Decompressor.write(chunk); - callback(); - } - - _flush(callback: plugins.stream.TransformCallback) { - // When the stream is ending, end the decompressor stream as well - this.bzip2Decompressor.end(); - callback(); - } -} +import { unbzip2Stream } from './bzip2/index.js'; export class Bzip2Tools { smartArchiveRef: SmartArchive; @@ -40,6 +11,6 @@ export class Bzip2Tools { } getDecompressionStream() { - return new DecompressBzip2Transform(); + return unbzip2Stream(); } } \ No newline at end of file diff --git a/ts/classes.smartarchive.ts b/ts/classes.smartarchive.ts index b888c65..49b5612 100644 --- a/ts/classes.smartarchive.ts +++ b/ts/classes.smartarchive.ts @@ -80,6 +80,7 @@ export class SmartArchive { streamFileStream.pipe(new plugins.smartstream.SmartDuplex({ objectMode: true, writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => { + const done = plugins.smartpromise.defer(); console.log(chunkArg.relativeFilePath); const streamFile = chunkArg; const readStream = await streamFile.createReadStream(); @@ -88,6 +89,10 @@ export class SmartArchive { await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath)); const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath); readStream.pipe(writeStream); + writeStream.on('finish', () => { + done.resolve(); + }) + await done.promise; }, finalFunction: async () => { done.resolve(); @@ -117,7 +122,6 @@ export class SmartArchive { stream.on('end', function () { next(); // ready for next entry }); - stream.resume(); // just auto drain the stream } ); tarStream.on('finish', function () { diff --git a/ts/plugins.ts b/ts/plugins.ts index 3167eb0..88713c2 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -20,6 +20,5 @@ export { smartfile, smartpath, smartpromise, smartrequest, smartunique, smartstr import * as fileType from 'file-type'; import * as fflate from 'fflate'; import tarStream from 'tar-stream'; -import unbzip2Stream from 'unbzip2-stream'; -export { fileType, fflate, tarStream, unbzip2Stream }; +export { fileType, fflate, tarStream };