fix(core): update
This commit is contained in:
parent
1113020e17
commit
75be95fe45
@ -26,7 +26,7 @@
|
|||||||
"@push.rocks/smartpromise": "^4.0.3",
|
"@push.rocks/smartpromise": "^4.0.3",
|
||||||
"@push.rocks/smartrequest": "^2.0.21",
|
"@push.rocks/smartrequest": "^2.0.21",
|
||||||
"@push.rocks/smartrx": "^3.0.7",
|
"@push.rocks/smartrx": "^3.0.7",
|
||||||
"@push.rocks/smartstream": "^3.0.18",
|
"@push.rocks/smartstream": "^3.0.25",
|
||||||
"@push.rocks/smartunique": "^3.0.6",
|
"@push.rocks/smartunique": "^3.0.6",
|
||||||
"@push.rocks/smarturl": "^3.0.7",
|
"@push.rocks/smarturl": "^3.0.7",
|
||||||
"@types/tar-stream": "^3.1.3",
|
"@types/tar-stream": "^3.1.3",
|
||||||
|
24
pnpm-lock.yaml
generated
24
pnpm-lock.yaml
generated
@ -21,8 +21,8 @@ dependencies:
|
|||||||
specifier: ^3.0.7
|
specifier: ^3.0.7
|
||||||
version: 3.0.7
|
version: 3.0.7
|
||||||
'@push.rocks/smartstream':
|
'@push.rocks/smartstream':
|
||||||
specifier: ^3.0.18
|
specifier: ^3.0.25
|
||||||
version: 3.0.18
|
version: 3.0.25
|
||||||
'@push.rocks/smartunique':
|
'@push.rocks/smartunique':
|
||||||
specifier: ^3.0.6
|
specifier: ^3.0.6
|
||||||
version: 3.0.6
|
version: 3.0.6
|
||||||
@ -597,6 +597,18 @@ packages:
|
|||||||
/@push.rocks/isounique@1.0.5:
|
/@push.rocks/isounique@1.0.5:
|
||||||
resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==}
|
resolution: {integrity: sha512-Z0BVqZZOCif1THTbIKWMgg0wxCzt9CyBtBBqQJiZ+jJ0KlQFrQHNHrPt81/LXe/L4x0cxWsn0bpL6W5DNSvNLw==}
|
||||||
|
|
||||||
|
/@push.rocks/lik@6.0.12:
|
||||||
|
resolution: {integrity: sha512-/vzlOZ26gCmXZz67LeM2hJ+aNM49Jxvf3FKxLMXHhJwffd3LcV96MYbMfKzKR/za/bh5Itf3a6UjLL5mmN6Pew==}
|
||||||
|
dependencies:
|
||||||
|
'@push.rocks/smartdelay': 3.0.5
|
||||||
|
'@push.rocks/smartmatch': 2.0.0
|
||||||
|
'@push.rocks/smartpromise': 4.0.3
|
||||||
|
'@push.rocks/smartrx': 3.0.7
|
||||||
|
'@push.rocks/smarttime': 4.0.6
|
||||||
|
'@types/minimatch': 5.1.2
|
||||||
|
'@types/symbol-tree': 3.2.5
|
||||||
|
symbol-tree: 3.2.4
|
||||||
|
|
||||||
/@push.rocks/lik@6.0.5:
|
/@push.rocks/lik@6.0.5:
|
||||||
resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==}
|
resolution: {integrity: sha512-rDSPVknpJDcY3yx4w3lNQfpRkBRuxYVIJKrkiODdH27X0z3+XU4e/iAWKySuLq6H7dAoxSHJDf6BrCO+3Y8kcw==}
|
||||||
dependencies:
|
dependencies:
|
||||||
@ -735,7 +747,7 @@ packages:
|
|||||||
'@push.rocks/smartpath': 5.0.11
|
'@push.rocks/smartpath': 5.0.11
|
||||||
'@push.rocks/smartpromise': 4.0.3
|
'@push.rocks/smartpromise': 4.0.3
|
||||||
'@push.rocks/smartrequest': 2.0.21
|
'@push.rocks/smartrequest': 2.0.21
|
||||||
'@push.rocks/smartstream': 3.0.18
|
'@push.rocks/smartstream': 3.0.25
|
||||||
'@types/fs-extra': 11.0.3
|
'@types/fs-extra': 11.0.3
|
||||||
'@types/glob': 8.1.0
|
'@types/glob': 8.1.0
|
||||||
'@types/js-yaml': 4.0.8
|
'@types/js-yaml': 4.0.8
|
||||||
@ -952,9 +964,10 @@ packages:
|
|||||||
through2: 4.0.2
|
through2: 4.0.2
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/@push.rocks/smartstream@3.0.18:
|
/@push.rocks/smartstream@3.0.25:
|
||||||
resolution: {integrity: sha512-o/ff/koLyfKL6FxoYURUZ5/CtprDG0K8aZmYdbpuIw6vnFgHCVYpARpQjiJW0Wb+lRh+crFJHEDO3jZhKCQO+w==}
|
resolution: {integrity: sha512-rnZ4Xbuhf6CbgRBjV3o7ddSIR2baWsbeEpV39meia2CfsyY9W/WKxBQrWJbrrv8w165ksSWm9Y9Vy0PmNKr5rQ==}
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@push.rocks/lik': 6.0.12
|
||||||
'@push.rocks/smartpromise': 4.0.3
|
'@push.rocks/smartpromise': 4.0.3
|
||||||
'@push.rocks/smartrx': 3.0.7
|
'@push.rocks/smartrx': 3.0.7
|
||||||
|
|
||||||
@ -1754,7 +1767,6 @@ packages:
|
|||||||
|
|
||||||
/@types/symbol-tree@3.2.5:
|
/@types/symbol-tree@3.2.5:
|
||||||
resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==}
|
resolution: {integrity: sha512-zXnnyENt1TYQcS21MkPaJCVjfcPq7p7yc5mo5JACuumXp6sly5jnlS0IokHd+xmmuCbx6V7JqkMBpswR+nZAcw==}
|
||||||
dev: true
|
|
||||||
|
|
||||||
/@types/tar-stream@3.1.3:
|
/@types/tar-stream@3.1.3:
|
||||||
resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}
|
resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==}
|
||||||
|
@ -38,7 +38,7 @@ tap.test('should extract existing files on disk', async () => {
|
|||||||
await testSmartarchive.exportToFs(testPaths.nogitDir);
|
await testSmartarchive.exportToFs(testPaths.nogitDir);
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.skip.test('should extract a b2zip', async () => {
|
tap.test('should extract a b2zip', async () => {
|
||||||
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2';
|
||||||
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl);
|
||||||
await testArchive.exportToFs(
|
await testArchive.exportToFs(
|
||||||
|
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartarchive',
|
name: '@push.rocks/smartarchive',
|
||||||
version: '4.0.11',
|
version: '4.0.12',
|
||||||
description: 'work with archives'
|
description: 'work with archives'
|
||||||
}
|
}
|
||||||
|
@ -38,18 +38,16 @@ export function unbzip2Stream() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var outlength = 0;
|
var outlength = 0;
|
||||||
const decompressAndPush = async (smartDuplexStream: plugins.smartstream.SmartDuplex) => {
|
const decompressAndPush = async () => {
|
||||||
if (broken) return;
|
if (broken) return;
|
||||||
try {
|
try {
|
||||||
const resultChunk = decompressBlock();
|
const resultChunk = decompressBlock();
|
||||||
smartDuplexStream.push(resultChunk);
|
|
||||||
if (resultChunk) {
|
if (resultChunk) {
|
||||||
//console.error('write at', outlength.toString(16));
|
|
||||||
outlength += resultChunk.length;
|
outlength += resultChunk.length;
|
||||||
}
|
}
|
||||||
|
return resultChunk;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error(e);
|
console.error(e);
|
||||||
smartDuplexStream.emit('error', e);
|
|
||||||
broken = true;
|
broken = true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -67,13 +65,17 @@ export function unbzip2Stream() {
|
|||||||
}
|
}
|
||||||
while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
|
while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) {
|
||||||
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
|
//console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer');
|
||||||
await decompressAndPush(this);
|
const result = await decompressAndPush();
|
||||||
|
console.log('pushing...');
|
||||||
|
await streamTools.push(result);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
finalFunction: async function (streamTools) {
|
finalFunction: async function (streamTools) {
|
||||||
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
|
//console.error(x,'last compressing with', hasBytes, 'bytes in buffer');
|
||||||
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
while (!broken && bitReader && hasBytes > bitReader.bytesRead) {
|
||||||
await decompressAndPush(this);
|
const result = await decompressAndPush();
|
||||||
|
console.log('pushing final...');
|
||||||
|
streamTools.push(result);
|
||||||
}
|
}
|
||||||
if (!broken) {
|
if (!broken) {
|
||||||
if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
|
if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely'));
|
||||||
|
@ -81,7 +81,7 @@ export class SmartArchive {
|
|||||||
objectMode: true,
|
objectMode: true,
|
||||||
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
|
writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamtools) => {
|
||||||
const done = plugins.smartpromise.defer<void>();
|
const done = plugins.smartpromise.defer<void>();
|
||||||
console.log(chunkArg.relativeFilePath);
|
console.log(chunkArg.relativeFilePath ? chunkArg.relativeFilePath : 'no relative path');
|
||||||
const streamFile = chunkArg;
|
const streamFile = chunkArg;
|
||||||
const readStream = await streamFile.createReadStream();
|
const readStream = await streamFile.createReadStream();
|
||||||
await plugins.smartfile.fs.ensureDir(targetDir);
|
await plugins.smartfile.fs.ensureDir(targetDir);
|
||||||
|
Loading…
Reference in New Issue
Block a user