BREAKING CHANGE(SmartArchive): Refactor public API: rename factory/extraction methods, introduce typed interfaces and improved compression tools
This commit is contained in:
@@ -1,51 +1,53 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { Bzip2Error, BZIP2_ERROR_CODES } from '../errors.js';
|
||||
import type { IBitReader } from '../interfaces.js';
|
||||
|
||||
import { Bzip2 } from './bzip2.js';
|
||||
import { bitIterator } from './bititerator.js';
|
||||
|
||||
export function unbzip2Stream() {
|
||||
/**
|
||||
* Creates a streaming BZIP2 decompression transform
|
||||
*/
|
||||
export function unbzip2Stream(): plugins.smartstream.SmartDuplex<Buffer, Buffer> {
|
||||
const bzip2Instance = new Bzip2();
|
||||
var bufferQueue = [];
|
||||
var hasBytes = 0;
|
||||
var blockSize = 0;
|
||||
var broken = false;
|
||||
var done = false;
|
||||
var bitReader = null;
|
||||
var streamCRC = null;
|
||||
const bufferQueue: Buffer[] = [];
|
||||
let hasBytes = 0;
|
||||
let blockSize = 0;
|
||||
let broken = false;
|
||||
let bitReader: IBitReader | null = null;
|
||||
let streamCRC: number | null = null;
|
||||
|
||||
function decompressBlock() {
|
||||
function decompressBlock(): Buffer | undefined {
|
||||
if (!blockSize) {
|
||||
blockSize = bzip2Instance.header(bitReader);
|
||||
blockSize = bzip2Instance.header(bitReader!);
|
||||
streamCRC = 0;
|
||||
} else {
|
||||
var bufsize = 100000 * blockSize;
|
||||
var buf = new Int32Array(bufsize);
|
||||
|
||||
var chunk = [];
|
||||
var f = function (b) {
|
||||
chunk.push(b);
|
||||
};
|
||||
|
||||
streamCRC = bzip2Instance.decompress(
|
||||
bitReader,
|
||||
f,
|
||||
buf,
|
||||
bufsize,
|
||||
streamCRC,
|
||||
);
|
||||
if (streamCRC === null) {
|
||||
// reset for next bzip2 header
|
||||
blockSize = 0;
|
||||
return;
|
||||
} else {
|
||||
return Buffer.from(chunk);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const bufsize = 100000 * blockSize;
|
||||
const buf = new Int32Array(bufsize);
|
||||
const chunk: number[] = [];
|
||||
|
||||
const outputFunc = (b: number): void => {
|
||||
chunk.push(b);
|
||||
};
|
||||
|
||||
streamCRC = bzip2Instance.decompress(bitReader!, outputFunc, buf, bufsize, streamCRC);
|
||||
|
||||
if (streamCRC === null) {
|
||||
// Reset for next bzip2 header
|
||||
blockSize = 0;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return Buffer.from(chunk);
|
||||
}
|
||||
|
||||
var outlength = 0;
|
||||
const decompressAndPush = async () => {
|
||||
if (broken) return;
|
||||
let outlength = 0;
|
||||
|
||||
const decompressAndPush = async (): Promise<Buffer | undefined> => {
|
||||
if (broken) return undefined;
|
||||
|
||||
try {
|
||||
const resultChunk = decompressBlock();
|
||||
if (resultChunk) {
|
||||
@@ -53,40 +55,39 @@ export function unbzip2Stream() {
|
||||
}
|
||||
return resultChunk;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
broken = true;
|
||||
if (e instanceof Error) {
|
||||
throw new Bzip2Error(`Decompression failed: ${e.message}`, BZIP2_ERROR_CODES.INVALID_BLOCK_DATA);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
let counter = 0;
|
||||
return new plugins.smartstream.SmartDuplex({
|
||||
|
||||
return new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
||||
objectMode: true,
|
||||
name: 'bzip2',
|
||||
debug: false,
|
||||
highWaterMark: 1,
|
||||
writeFunction: async function (data, streamTools) {
|
||||
// console.log(`got chunk ${counter++}`)
|
||||
bufferQueue.push(data);
|
||||
hasBytes += data.length;
|
||||
|
||||
if (bitReader === null) {
|
||||
bitReader = bitIterator(function () {
|
||||
return bufferQueue.shift();
|
||||
return bufferQueue.shift()!;
|
||||
});
|
||||
}
|
||||
while (
|
||||
!broken &&
|
||||
hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)
|
||||
) {
|
||||
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
|
||||
|
||||
const threshold = 25000 + 100000 * blockSize || 4;
|
||||
while (!broken && hasBytes - bitReader.bytesRead + 1 >= threshold) {
|
||||
const result = await decompressAndPush();
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
// console.log(result.toString());
|
||||
await streamTools.push(result);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
finalFunction: async function (streamTools) {
|
||||
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
|
||||
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
||||
const result = await decompressAndPush();
|
||||
if (!result) {
|
||||
@@ -94,10 +95,11 @@ export function unbzip2Stream() {
|
||||
}
|
||||
await streamTools.push(result);
|
||||
}
|
||||
if (!broken) {
|
||||
if (streamCRC !== null)
|
||||
this.emit('error', new Error('input stream ended prematurely'));
|
||||
|
||||
if (!broken && streamCRC !== null) {
|
||||
this.emit('error', new Bzip2Error('Input stream ended prematurely', BZIP2_ERROR_CODES.PREMATURE_END));
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user