diff --git a/package.json b/package.json index 0fcc662..86ea350 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartrequest": "^2.0.21", "@push.rocks/smartrx": "^3.0.7", - "@push.rocks/smartstream": "^3.0.18", + "@push.rocks/smartstream": "^3.0.25", "@push.rocks/smartunique": "^3.0.6", "@push.rocks/smarturl": "^3.0.7", "@types/tar-stream": "^3.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54a1b83..eac3201 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ dependencies: specifier: ^3.0.7 version: 3.0.7 '@push.rocks/smartstream': - specifier: ^3.0.18 - version: 3.0.18 + specifier: ^3.0.25 + version: 3.0.25 '@push.rocks/smartunique': specifier: ^3.0.6 version: 3.0.6 @@ -597,6 +597,18 @@ packages: /@push.rocks/isounique@1.0.5: resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==} + /@push.rocks/lik@6.0.12: + resolution: {integrity: sha512-/vzlOZ26gCmXZz67LeM2hJ+aNM49Jxvf3FKxLMXHhJwffd3LcV96MYbMfKzKR/za/bh5Itf3a6UjLL5mmN6Pew==} + dependencies: + '@push.rocks/smartdelay': 3.0.5 + '@push.rocks/smartmatch': 2.0.0 + '@push.rocks/smartpromise': 4.0.3 + '@push.rocks/smartrx': 3.0.7 + '@push.rocks/smarttime': 4.0.6 + '@types/minimatch': 5.1.2 + '@types/symbol-tree': 3.2.5 + symbol-tree: 3.2.4 + /@push.rocks/lik@6.0.5: resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==} dependencies: @@ -735,7 +747,7 @@ packages: '@push.rocks/smartpath': 5.0.11 '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartrequest': 2.0.21 - '@push.rocks/smartstream': 3.0.18 + '@push.rocks/smartstream': 3.0.25 '@types/fs-extra': 11.0.3 '@types/glob': 8.1.0 '@types/js-yaml': 4.0.8 @@ -952,9 +964,10 @@ packages: through2: 4.0.2 dev: true - /@push.rocks/smartstream@3.0.18: - resolution: {integrity: sha512-o/ff/koLyfKL6FxoYURUZ5/CtprDG0K8aZmYdbpuIw6vnFgHCVYpARpQjiJW0Wb+lRh+crFJHEDO3jZhKCQO+w==} + /@push.rocks/smartstream@3.0.25: + resolution: {integrity: sha512-rnZ4Xbuhf6CbgRBjV3o7ddSIR2baWsbeEpV39meia2CfsyY9W/WKxBQrWJbrrv8w165ksSWm9Y9Vy0PmNKr5rQ==} dependencies: + '@push.rocks/lik': 6.0.12 '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartrx': 3.0.7 @@ -1754,7 +1767,6 @@ packages: /@types/symbol-tree@3.2.5: resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==} - dev: true /@types/tar-stream@3.1.3: resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==} diff --git a/test/test.ts b/test/test.ts index 5722b37..29d7c1e 100644 --- a/test/test.ts +++ b/test/test.ts @@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => { await testSmartarchive.exportToFs(testPaths.nogitDir); }); -tap.skip.test('should extract a b2zip', async () => { +tap.test('should extract a b2zip', async () => { const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl); await testArchive.exportToFs( diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 8c4e35c..d42e6ed 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartarchive', - version: '4.0.11', + version: '4.0.12', description: 'work with archives' } diff --git a/ts/bzip2/index.ts b/ts/bzip2/index.ts index 4c398a0..283599a 100644 --- a/ts/bzip2/index.ts +++ b/ts/bzip2/index.ts @@ -38,18 +38,16 @@ export function unbzip2Stream() { } var outlength = 0; - const decompressAndPush = async (smartDuplexStream: plugins.smartstream.SmartDuplex) => { + const decompressAndPush = async () => { if (broken) return; try { const resultChunk = decompressBlock(); - smartDuplexStream.push(resultChunk); if (resultChunk) { - //console.error('write at', outlength.toString(16)); outlength += resultChunk.length; } + return resultChunk; } catch (e) { console.error(e); - smartDuplexStream.emit('error', e); broken = true; return false; } @@ -67,13 +65,17 @@ export function unbzip2Stream() { } while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) { //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer'); - await decompressAndPush(this); + const result = await decompressAndPush(); + console.log('pushing...'); + await streamTools.push(result); } }, finalFunction: async function (streamTools) { //console.error(x,'last compressing with', hasBytes, 'bytes in buffer'); while (!broken && bitReader && hasBytes > bitReader.bytesRead) { - await decompressAndPush(this); + const result = await decompressAndPush(); + console.log('pushing final...'); + streamTools.push(result); } if (!broken) { if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely')); diff --git a/ts/classes.smartarchive.ts b/ts/classes.smartarchive.ts index 6844636..2742047 100644 --- a/ts/classes.smartarchive.ts +++ b/ts/classes.smartarchive.ts @@ -81,7 +81,7 @@ export class SmartArchive { objectMode: true, writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => { const done = plugins.smartpromise.defer(); - console.log(chunkArg.relativeFilePath); + console.log(chunkArg.relativeFilePath ? chunkArg.relativeFilePath : 'no relative path'); const streamFile = chunkArg; const readStream = await streamFile.createReadStream(); await plugins.smartfile.fs.ensureDir(targetDir);