fix(core): update

This commit is contained in:
Philipp Kunz 2023-11-13 20:00:28 +01:00
parent 1113020e17
commit 75be95fe45
6 changed files with 30 additions and 16 deletions

View File

@ -26,7 +26,7 @@
"@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrequest": "^2.0.21", "@push.rocks/smartrequest": "^2.0.21",
"@push.rocks/smartrx": "^3.0.7", "@push.rocks/smartrx": "^3.0.7",
"@push.rocks/smartstream": "^3.0.18", "@push.rocks/smartstream": "^3.0.25",
"@push.rocks/smartunique": "^3.0.6", "@push.rocks/smartunique": "^3.0.6",
"@push.rocks/smarturl": "^3.0.7", "@push.rocks/smarturl": "^3.0.7",
"@types/tar-stream": "^3.1.3", "@types/tar-stream": "^3.1.3",

24
pnpm-lock.yaml generated
View File

@ -21,8 +21,8 @@ dependencies:
specifier: ^3.0.7 specifier: ^3.0.7
version: 3.0.7 version: 3.0.7
'@push.rocks/smartstream': '@push.rocks/smartstream':
specifier: ^3.0.18 specifier: ^3.0.25
version: 3.0.18 version: 3.0.25
'@push.rocks/smartunique': '@push.rocks/smartunique':
specifier: ^3.0.6 specifier: ^3.0.6
version: 3.0.6 version: 3.0.6
@ -597,6 +597,18 @@ packages:
/@push.rocks/isounique@1.0.5: /@push.rocks/isounique@1.0.5:
resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==} resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==}
/@push.rocks/lik@6.0.12:
resolution: {integrity: sha512-/vzlOZ26gCmXZz67LeM2hJ+aNM49Jxvf3FKxLMXHhJwffd3LcV96MYbMfKzKR/za/bh5Itf3a6UjLL5mmN6Pew==}
dependencies:
'@push.rocks/smartdelay': 3.0.5
'@push.rocks/smartmatch': 2.0.0
'@push.rocks/smartpromise': 4.0.3
'@push.rocks/smartrx': 3.0.7
'@push.rocks/smarttime': 4.0.6
'@types/minimatch': 5.1.2
'@types/symbol-tree': 3.2.5
symbol-tree: 3.2.4
/@push.rocks/lik@6.0.5: /@push.rocks/lik@6.0.5:
resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==} resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==}
dependencies: dependencies:
@ -735,7 +747,7 @@ packages:
'@push.rocks/smartpath': 5.0.11 '@push.rocks/smartpath': 5.0.11
'@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartpromise': 4.0.3
'@push.rocks/smartrequest': 2.0.21 '@push.rocks/smartrequest': 2.0.21
'@push.rocks/smartstream': 3.0.18 '@push.rocks/smartstream': 3.0.25
'@types/fs-extra': 11.0.3 '@types/fs-extra': 11.0.3
'@types/glob': 8.1.0 '@types/glob': 8.1.0
'@types/js-yaml': 4.0.8 '@types/js-yaml': 4.0.8
@ -952,9 +964,10 @@ packages:
through2: 4.0.2 through2: 4.0.2
dev: true dev: true
/@push.rocks/smartstream@3.0.18: /@push.rocks/smartstream@3.0.25:
resolution: {integrity: sha512-o/ff/koLyfKL6FxoYURUZ5/CtprDG0K8aZmYdbpuIw6vnFgHCVYpARpQjiJW0Wb+lRh+crFJHEDO3jZhKCQO+w==} resolution: {integrity: sha512-rnZ4Xbuhf6CbgRBjV3o7ddSIR2baWsbeEpV39meia2CfsyY9W/WKxBQrWJbrrv8w165ksSWm9Y9Vy0PmNKr5rQ==}
dependencies: dependencies:
'@push.rocks/lik': 6.0.12
'@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartpromise': 4.0.3
'@push.rocks/smartrx': 3.0.7 '@push.rocks/smartrx': 3.0.7
@ -1754,7 +1767,6 @@ packages:
/@types/symbol-tree@3.2.5: /@types/symbol-tree@3.2.5:
resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==} resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==}
dev: true
/@types/tar-stream@3.1.3: /@types/tar-stream@3.1.3:
resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==} resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}

View File

@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => {
await testSmartarchive.exportToFs(testPaths.nogitDir); await testSmartarchive.exportToFs(testPaths.nogitDir);
}); });
tap.skip.test('should extract a b2zip', async () => { tap.test('should extract a b2zip', async () => {
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl); const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
await testArchive.exportToFs( await testArchive.exportToFs(

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartarchive', name: '@push.rocks/smartarchive',
version: '4.0.11', version: '4.0.12',
description: 'work with archives' description: 'work with archives'
} }

View File

@ -38,18 +38,16 @@ export function unbzip2Stream() {
} }
var outlength = 0; var outlength = 0;
const decompressAndPush = async (smartDuplexStream: plugins.smartstream.SmartDuplex) => { const decompressAndPush = async () => {
if (broken) return; if (broken) return;
try { try {
const resultChunk = decompressBlock(); const resultChunk = decompressBlock();
smartDuplexStream.push(resultChunk);
if (resultChunk) { if (resultChunk) {
//console.error('write at', outlength.toString(16));
outlength += resultChunk.length; outlength += resultChunk.length;
} }
return resultChunk;
} catch (e) { } catch (e) {
console.error(e); console.error(e);
smartDuplexStream.emit('error', e);
broken = true; broken = true;
return false; return false;
} }
@ -67,13 +65,17 @@ export function unbzip2Stream() {
} }
while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) { while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer'); //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
await decompressAndPush(this); const result = await decompressAndPush();
console.log('pushing...');
await streamTools.push(result);
} }
}, },
finalFunction: async function (streamTools) { finalFunction: async function (streamTools) {
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer'); //console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
while (!broken && bitReader && hasBytes > bitReader.bytesRead) { while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
await decompressAndPush(this); const result = await decompressAndPush();
console.log('pushing final...');
streamTools.push(result);
} }
if (!broken) { if (!broken) {
if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely')); if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));

View File

@ -81,7 +81,7 @@ export class SmartArchive {
objectMode: true, objectMode: true,
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => { writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
const done = plugins.smartpromise.defer<void>(); const done = plugins.smartpromise.defer<void>();
console.log(chunkArg.relativeFilePath); console.log(chunkArg.relativeFilePath ? chunkArg.relativeFilePath : 'no relative path');
const streamFile = chunkArg; const streamFile = chunkArg;
const readStream = await streamFile.createReadStream(); const readStream = await streamFile.createReadStream();
await plugins.smartfile.fs.ensureDir(targetDir); await plugins.smartfile.fs.ensureDir(targetDir);