From 780db4921e14a5ece6f4b1a7ecf9a97c71f5680c Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 18 Aug 2025 01:29:06 +0000 Subject: [PATCH] feat(classes.smartarchive): Support URL streams, recursive archive unpacking and filesystem export; improve ZIP/GZIP/BZIP2 robustness; CI and package metadata updates --- .gitea/workflows/default_nottags.yaml | 6 +- .gitea/workflows/default_tags.yaml | 12 +- .gitignore | 7 +- .../document_symbols_cache_v23-06-25.pkl | Bin 56731 -> 71489 bytes changelog.md | 25 +- npmextra.json | 2 +- package.json | 9 +- readme.md | 489 +++++++++------- test/plugins.ts | 8 +- test/test.ts | 27 +- ts/00_commitinfo_data.ts | 2 +- ts/bzip2/bititerator.ts | 77 +-- ts/bzip2/bzip2.ts | 524 +++++++++--------- ts/bzip2/index.ts | 16 +- ts/classes.archiveanalyzer.ts | 25 +- ts/classes.bzip2tools.ts | 2 +- ts/classes.gziptools.ts | 18 +- ts/classes.smartarchive.ts | 102 +++- ts/classes.tartools.ts | 30 +- ts/classes.ziptools.ts | 31 +- ts/paths.ts | 2 +- ts/plugins.ts | 12 +- tsconfig.json | 8 +- 23 files changed, 812 insertions(+), 622 deletions(-) diff --git a/.gitea/workflows/default_nottags.yaml b/.gitea/workflows/default_nottags.yaml index 9f4e743..0bae651 100644 --- a/.gitea/workflows/default_nottags.yaml +++ b/.gitea/workflows/default_nottags.yaml @@ -6,8 +6,8 @@ on: - '**' env: - IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci - NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git + IMAGE: code.foss.global/host.today/ht-docker-node:npmci + NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}} NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}} NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}} @@ -26,7 +26,7 @@ jobs: - name: Install pnpm and npmci run: | pnpm install -g pnpm - pnpm install -g @shipzone/npmci + pnpm install -g @ship.zone/npmci - name: Run npm prepare run: npmci npm prepare diff --git a/.gitea/workflows/default_tags.yaml b/.gitea/workflows/default_tags.yaml index e27ad69..821d33f 100644 --- a/.gitea/workflows/default_tags.yaml +++ b/.gitea/workflows/default_tags.yaml @@ -6,8 +6,8 @@ on: - '*' env: - IMAGE: registry.gitlab.com/hosttoday/ht-docker-node:npmci - NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git + IMAGE: code.foss.global/host.today/ht-docker-node:npmci + NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}} NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}} NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}} @@ -26,7 +26,7 @@ jobs: - name: Prepare run: | pnpm install -g pnpm - pnpm install -g @shipzone/npmci + pnpm install -g @ship.zone/npmci npmci npm prepare - name: Audit production dependencies @@ -54,7 +54,7 @@ jobs: - name: Prepare run: | pnpm install -g pnpm - pnpm install -g @shipzone/npmci + pnpm install -g @ship.zone/npmci npmci npm prepare - name: Test stable @@ -82,7 +82,7 @@ jobs: - name: Prepare run: | pnpm install -g pnpm - pnpm install -g @shipzone/npmci + pnpm install -g @ship.zone/npmci npmci npm prepare - name: Release @@ -104,7 +104,7 @@ jobs: - name: Prepare run: | pnpm install -g pnpm - pnpm install -g @shipzone/npmci + pnpm install -g @ship.zone/npmci npmci npm prepare - name: Code quality diff --git a/.gitignore b/.gitignore index ef13c79..14fcd33 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ # artifacts coverage/ public/ -pages/ # installs node_modules/ @@ -17,4 +16,8 @@ node_modules/ dist/ dist_*/ -# custom \ No newline at end of file +# AI +.claude/ +.serena/ + +#------# custom \ No newline at end of file diff --git a/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl b/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl index 6745bd5215166f70d70e4a565ddaeb15d3f4a835..b6ffaf6d6ab6ce8d26d9392527290d7dbe563b7d 100644 GIT binary patch literal 71489 zcmeHQeUKbSbr;s1bay(PWLcJF`D@NVD2fm&d=!LGfQw3{QYj|{Dk&=O zP4|3sPxtKJY2RX!y+3-d-*(UR{@&}?uU~h+eqGnQx|Y0T3I5I3hiXQ}9my0-*R|Y~ zJ8PCJW;tW$&swR9yZwk+aIJdH*tuhB=h%d`D?2tZwR_aenVIzHj$Kw}d}1n>oybkp zPuGX=SzSeQ7RvXZz~>xb&#BjzWvz;tFVt&G>b0&j`C_&{-CgT0o5dNcjs`;3TahUh-AcKdsg%lfd%4|X_fGfPeOQk_Y8)PQ zS@0;#t*KV+H4RIwI}N{k&V;vB>ULMHr%=k6*jDdXF0GdH_1Yb|e8Ji?GBPq-tc+yM ziaAm!xo*L7-I2MfYp2Sk%o%qi%90U1tLi?W_J&%&nRZKsYQ?(Stl0J1;TFB1zibuY zH=N~<-H_0jY83(MuB|q+Su1-eUj{~%%6ZH6L0&F^yh8vPbHftINb3#QfH~l(4ItYa z?JoO9eB0(018Q%vx7&lQLeCzyxB95`d8ix#?rZQE74X2^{SqEX3-Q3*6>Y#{iUyBf z50CrG1q~cy0vwpTQGx?$Asm>)UP>(@qFK1`C@4U#n8ZFaD1d^wT@olr z3qir$`Zj=a4-F_QI4H+)fKdXnT>u1g*GYgNEd&H}Yufx?ZzI zzyotPOL!nH!~=8fP$GFEJgoC`rE=wD=}4eOmI;H*2?38+L2jvp2bNeB3^JJGgACd> zFq(B`OJJOE;TsShei91z1ywP(Q+jy3#6Tc?F4+k78XB-y=JKGNj#P`D{pCX<;(Z1{ z=m{X1gQ^-mN|=7&I#{7UrNM|b06Rkb{xzdmQab8*w6+kd6EW|mFLR&ij~}P?iU2bft=iN z#1fd}9jCFPdH@Qq4tgc#rX;-MCBjki<7A5JPiXM&0~yRF$=_)K56rzx!XsWHls`H= zzC?sa*elij@gV^Z%)zup&{FXd?ZD&fGTOq+AmcU#%Go=%c z@6o`q!h<8JzhnhKFn6B>NW8=~P=d|H=m2@129S*rKvJdR&>$E-dDk8sHZo?RkTx@C z{BG1Q*n>F$W>cVMZOF-Tqb@-W(+jPJ$pQxqoo(_-issV!i>j?|E}M5;<4tGJ`Jcu7RAy)GFc9JeQ2em@m4Re!{E{K=7%GU+X zQgA_2nG?ASNMaAOInMB*-ynqkp?vn6{m$<$?_-r7}VZ5^=$=GtWIJVgKu3O=bJeG5^eNvEe< z2pY|LYh>BwXt83R(O6Lxkg9O%eim}eB=swnz#O+N*?SsVuT}4v{vIsMt&`pmFR>a* zFa_{g1c++bdyW8o6Xcv!Wd4|-(dfJ*^M*Oy1TAL5cZKf|J8710dsH-xMMu~jAa|i7 z>>kL;jxa0{I>IoAn~OqouWttSS%|L?CnU$AO4yY!cTf`ac!|)-@O4T-|1KE^R)SG* zcLN8{odPhJ+b01NFA=&2NrCw(0Wh?k=^F$sW}RtOu3Vbsdv&AJZm}b+1@?siBfJ-1YDq%o0|w)%;aYEjO8CINzyn|z`sy(jzLbA94rx9eVDs}Nshm? zgq|z$6{6L58&v5JMmx;OYkTn$I00hQn_DOa<_H=5J!Q+CgA+5FaeKdj$;%*zTU?{d zZDI+`g_bZWCijwI(pxPTk_6>e0Tj&1(}Q@4U>mGM3d$?#KsjfnlR!BjfP%SUNmAk^ z+60P201EBYU>`w?nY!)8xo^S|gPVYHp`OJYn*7lYkF@ZJ0p`Lj&2*lJ3uK54xcTBt z!NPfxey4XzfCY21bR#W<1#`i~mvAcZTZFKbs^yFaOLQ-#xB-5v01M{i^=zbtuwbqo zu)LEDmVVcRMbv~MC^rkBU`}4SMOp|7<^oMfC@UWz1ErUN5;6NlM1}=KFo!GEQKf>k z5E0C^L#aGNhDZ-M(-O4oCIJx4ZIS>%S_lZ{a81cGH3>!Jb98_t4?CL$Krkl*g0v72 z%(VlMFA@M^QLTd5D_ZP~us0sB+UxC`?MeGq`!4%Idj>WJzuJC_UAI4If873@{YCq0 z_IK=`*#GV=lsNt1m~z%*g6o6Mc1&<9mva!4W|fHH|USRZ$DYCV&sIW@$nry$9HFUPMPVIqR=0?*y zCMGARa-+>fPOy6}HE@FSeQ5ghr<@<)hbDZ2^F8dqfC|pH zpr9C2!TCoZ**(s;;bXej`3`@Q@S<_{qRXTgaWbfO=bsl>?I%zTBLq1A0RK?+Bj;!M z@D}>@YKegrYF+74_Cme3Iuf}8c5z;uuVcss#({X$1%VCA)oj?IW&KQ|Grn^ z9w&pCIPGD>FIvn7=LL9R)%g!dYlCyubRnNHjeW+0r*DbyAR6PL z8T3ch=yUS@I`sMXGWwi+McdHtUfSAD5JmqLQS@=<~A2%ey5K;3%umV4*}g&w9XlP>D8wWRqVVw993CDh>d1g zv0cpyPKHfu>TbZlms!Ed;1F&TE4C4`;u?_^7`4Rj2vNwRK|Gp4e2W@!PKNj%YQ*{X zGUA-vzENU|5b~_h!Rw`@dtn%KorF?e(ke~*o*d_uGG-8osJm`^lU^6nDm{RjVQ;n0s%hx6z zJVeNY1H$pIJ%;ICC}s?pvFzJ#TL zO~-BO1!oD--U~Lc5DW*jl6lU|R}8emQvN8!z!uJTOS(x6A0u!@qnl2u`+(IzH)*PI za;TdYGi2Sz-o08Nev%fq2>c8LqG;KACjxJ zj}IRv$!870fn1K2$ImZfs`hEtGj6%Dq7C z3W(??tXhJeQYg&~3GIpNx0i}zRJ_0*W3X5GeJD+oC4OOMU9_9^-E26J*{%?v`4V+K~Hj zAVf`1_C3bAlDB|K4d*GY<@hq~$s{Yy);|wb=L**Oq0qAybMpj}(NVje^HFA5NA0Su z^gwP5KGX;@7#}RIg*X3(-rYlOTg#>tbqX<>3T*oz)c z3x9}%8kL9fU+6~6T1p7)85oY9A6hC&R%xwc91R8XUiD6FYuE z$c`hbaH7B!HWBo=!Wrgo+(;pp*wrvR`l!ezRRGRrm~>Xnq>nN(#O72pNmGrJA@W|V zKSx8y2o2M$;(bKjFiu8yr*ai9xtg$Cu(XS?GY{ayTev?ox-~9w6>qVDKn0#?d^qPB zPvjNv4>PiM#k+B}nDQ3}NYSgV8JOP18X+$)y@!$EAb?t4G}Sm6cQXb4rK>6BWiz3? zyi!0j@U_HzCjPQ~I1ypNq*88v22{`CVit5xV8i&OqvpjzJNY}+OyguYD5hpway4No zgp_HcgiK59a}lf&}$o4lZV@c;I_@Q^G&QlH%RS#jFBPk zty;z!R||&Tm!lX^&AUuO8M`aTZ%aedKJ?G|V`mY%I*h5t$cX3GzhFR^cG$LkzG8TB zjRh^h7k+}xqS0Wx0+%$J;0=r&Yb7pm@@XBHlB)^RF;XrS3AuF4^^`nn;025UtkO2a zh6Nd`oWd0be`StTC!V51;?x)%eBzrm+0XJ<*|vFNsWch$Ib(=pP=WEw*tc(=F$ha8 z-tE5_n>#M|=zjH=*d75r%9qiMpw=E^^Ti0R^F}I_GTbxyxj8s^web;!8Opno)MX6l zSLhUz&DMFToUTDA5^`JOZ;yJDTOmYJHVJwqp<#k5Z&dN2_IJteZ@){v0pCDd}^KFb6wh)D?88n81Q*N91-WY^Had+JYnf{{#N+s6hn9K;mmQm9|qtZX(v%f~izZVt`` zn-!Q9jGW&N!&hvKV*^4kL1@+dLeRUx(8H5#rAzTVi@cBs%AtREBrFU zSXkSYl^3=%#FqyB4iUe^i+p&JurQ#vg9|6Np)jWC{ht>b7PUqY){y55^TP`fq`b?I zVH;bvH0x=Sr%mk_EY%w2&@kI^;G^hEaQyl|C>d2XysKRl*cP2Pp((O}$+z$vwL>-9 z?Jm^;E4(fyI+8^t{BR?(MKpOiC6qDW_#)kI&`oH3J>0uwNn9j=z1{*h2h-mx6y46o zD#dOr+D^_5Z~x`;MX4@zRK{K0zlax_Lw^yePahLVG)g zUSz4R(Il^9=mm>j;ePlN!NRETBUYcp@J)qAqo&ReA&4zLtYhf;km6?xW)K}i&j+-T z~6g3NS^``ZK)oh6(syVeWv#cPbo?_6G-H)cwd?)F40q^dq|M!>49fXwE+| z5TsBd!qKK*6X2ejW7GYG?L?1MR1H)9l#)eMX=cxKpZ7%Aa8a--uECyZnBfX(#WpKO z#w;3X3qA@3D{IWcKCV2RureyJ1mGH#l2I+r!8LeB!MtFVU89sU_?z8WbMNy?GQRkY z^$HoHNyYfWk8gJ7zy5%XpW;-7UHIKu=!kBcKn2)Y=!nV;9+KzXjFIKMC{&$=j>JXI z&iq$Y){?=|ng9BGo;wR2MrWY|2BYQ|IszrV(D^U3;{;=pxk6NCA}mHlzl<7X!8#Q` zJu4f?IE+M(xF~O-7UtWUtx&hId98eM_6|nYo}A^?fkt^_(ccT(AB`o7{gM7aK9Nlj|)eRTgDC3=+ zG|RU=$|je=`GR2YS%^TBD0#Y=RbRB+d(VK`wR)~E@Q!KZ@J-xe_s&9ifeZxYF6-gg z1?ED((&*r|Y8|}3fez-?)=5VvI2mp~DPHFs@ zQW^gnT`#C@_12yauxk$dyW~N%(P4OE)-3yX1bh7*-J+!Njebh_OMk>WyIX-xrf>m` z7vm^$c{Z*+4B;f?Zipx51xf>}1rA0;f*M5u@ndeYKG{5qUSC!fPqEAVuo>6_;6bD@ zY~Jl#QxU%SER%b(;zoI)r)c`}26bOH8uVpeP43H_3^&lXt}kDw?#rC4?aRs4gueVM z%D&tSQMRZ;7`(Xp@)EjVdTMJpcI&z`AHZ36^Hw>;;D;V5h2e+IGBZ+pwBv%u3!g>^Ds@2U+bUo!oHok8l{H1^2V0;I0AF7rMzrK+j8UTHlHDSRWceNZ) zs;RC!rS7_i8+2V>O?HoP@(n@P4Fikyp{>I7lh`ngMShUApkFfnIawRVD!CeNV+al- zj#2j5eS~s+D1;KkCB7z~b5oTPj$3?$^8#Cmwxh6-r57Y_0i4;d<_ss_0T3%gL?u@X zFS&pj1Sw~Z5pw20fFv+R5%srhy{c-t6?Bxuc>}%QKgdoxg&L;=;M9C*cCqP`tds(O z32q3F2quFgJbl3@GO-d&EwcSs(jySR;LtmpbX_t#Jmb(m&5oLxwKJg={_IR{N-O%T z*qZQ(VRND`^WMN)Dk@w}Z4);f0zjd6 zqx<7FSfTYipQ7h^zy+s3!9q#6vy_!?C()pvQlrnw+6#gC_cHpNEV!OY(YFcF_b&t% zQQc#tAp*SxJBGjwV#0aCXc(XYakOhn{r);NPrt zzNo0U3HL>Tb@u;i|CAPp&c#_wiqt0t@jK#hNX$))_qEAAf#2kl%33+A%q}YfFCUkB zY#CPdgQ+qD0kt439>%XD?A%vabW`5lRMgkoM);#xe({DR&~D+Si@x&n8rQk6Fb{6j z0K)k&dQ+3JA9RcpAy?U`0Zu@1%(G?)}&;)L5;78X(`9H-!*SSO%p7EYp9%2g{Ivk`Lq zBetN~OOoTTal=?JpIN=YGVmYMi8B=hW z+4jxX?TyE)&hxOa;`|(wr4UBPS&hkcm~6#l5|cxi+=qo+w_|cYCOJ%8OkRh{+c9|` zCQn0RovwT5nas!PkJW3}RooG8RmgHv=(F}B@1-j4_9M6frCu|3Or$5q#?n(eCUUuS zc5-Yow|io2r!_V)z9T!49^G9(UB?}F!!Ld_7~;v3@t4X((5`f4d6#qE(;ns`X}U6R*Rv%Ju0b)!I@E zYT?Pazk#1IeTDO7{CG*VS{rb!0(gBPivGR+^PIneFN3mOtnA9F^S25zY|F3pWbAw) zTegb)_c+f%!P<(5%eL-(5lD8A^B3?j-RpdbKS_8|H+vBr%@$s?oRRTp_E#5I?dwqO zG>EkGEd1jyLmtK1|DKVNk=bHpBx_d8kwVFJ!L99%V0^h$xs*BMj(F#;xR-rI!|uAh z#rY;ad9Ze{6;B(kU8Q?dwTf)<;O%?363B;u?lLvHy=rvVG(cBVjg$LdWEGWhHWy{Zfaf|W6VCIH1k!xrlP5 zLk;Oz14wx_*k~zCA2=Crt7`+KFQbHX9Y7k|BMSp{Tn*|(15kN28B|WjgPm;v^$r41 ziPY%Lt#7KQM#tc}&VEdeK>}`wA#Xb1ja?3E2+Q+GTpO7^uqJ!V>F zdNemWIXXF+-7&d)*N)v&*_~tKle==cU89miq6c>7c>AUo;elAH^+4Fr-09aidHm3M zAad{}4i;Y7rK~KdaYsRh!yR!C#3N8p@<6141RjVCe1Hcc%bz5?sGGeAl)3OCPR8>H zg~e5ygKC}!;$is5SLU~0k_RHfUUWcI@WF$%1B-J&2oD>sty7Cpow*-8Y*)fQK=k9E zEMb7Fx)VirnHt?5HM$QoG7gnWba^#6TP`q#ldpwp3|$l#uW5aI{o7ghi|y)Hw`(11 zz;)6KUdhO46D!-*xEgDhXIyv@poG&i_UA{VG+5zoMU+KLdCknD$Z2wrqJA?Mw62_>CjBBhUc2-(R_js&2BG_ zw}{|?S{{!v1R(%}Ade3)@^T%5nrfUJ%466|diPd~XNskBMXC2LSGVg@!sT1wsI?oP%)pOJfYh&HZ9FOq-PdPFhiC)rT*vDUXYup@fVKEtCGqJGqj*H@{# z5hoAnn4Vk>P0oPnWFUTz8pIQDAWj5PGssLZ5C^25!^wB*KuoTNeja)dpKm>g*mGWs zkLBKu&D&dD2fFeX@-ZET$<n_^7ezMk;GZG zByuvwIaY$0T&+Eln5G7?>A9@KK$M4PPCldqF}WIE=tD1wMQRXFmMzJvqOoyu3`E)R z=HwGP5Rvr zy*&zLc&z8I^4z`7J2CqKOwj-D{1GP4V)7MCzJtk6F!?V`aQf@?V}j!3Y{p~%jyGD2C%$-wX@NXu& z+nU;GrpMEnF=-D(-$6KvF}B!U5Z&5c5Pt|wpI+sB7(aBoApQWp#L25ywwslOT@X(z z$Y9yTcR_pr3QD^mo&XZ;g7_eOz%Gan@h1r{>R~SekCgDDK1N25)-#K%_6exw?Sgne z{Np<{k6qF(2!X}858~7K=)u~pi`xewJ#M(RUMEg`FNBA%)_oWSdzl*Sel^$w4Zzk^ z<7Aw~w*%~FTMD-GXYe?-)bGK%DthVBy;iR7MDbj%#&ek(Pfo_Ly-MLtu7=Ch?ZERZ l6nF+Z-CEq!o+N0jH;#M?9_;)%CSQgG)jR47PrX-D{|i|Q9RmOW literal 56731 zcmeHQYm6MnaXyKp#5*1zq9~c7EORJQguZM?&_ah`0}U!ejxrg+Zt&)^=PbEaihqK z3emJ1*4?n^moIpQdbIVhTZz0@)7iCS$9>y({77m0E_brH-Q7`qpt!SCn%Xvb-v@W@ z+%>hWHPITuXDzI{(@=l-6h3F4cuuQ%Q^~8lLnn*TPnd zB6w##{_ny{x$3o!4L1jiz8kv5x)-*N4K$hyJZJ@V{bk3#>@PodL!;4L7I_t~STEPA zk0np?heefPsIs)-uTZ%=puuAJtV${LslZ7u-s}sGF7g4yT``H zrmOX_l3RDjDzzx8cu_Pq(}?^+SSy~7#&nX5<&&zF0P3%44!e_4tphe9UZmHyz4wge8R4pufQ3CSK1myb&$e0^7K*qAclnj^y ziuwSuzs_Ii-;HnEFwcPc8~v^Rrujn8AN4mUs0_uZ90uy=;IWP1fw_+wcwm{r19P|b z0FPZfcnrpPoDM5FaNJMez}(#i99X8{z#NWJZuU6D1IKbHfcI71%H=B{0Ht7}V`dWp z1#=G=pkSE-1#_!=0Lr60pe&J~9LEX93dmLh2rc{<5p zPEInY+rVtLuv7!(L^9tp^6(Q-c_?j)x$VZovo)5b%onlEU_Z_S_R^^`2&co1YOH@H zkWjp*0EERUBtGN@4M=DW%-z-lNPdYB{Ik|0P^P2hK^rK@A+t9WqeK)>zX6E_sy)m8BQMa@}h`aK;Iv zc#ty(4y}PXIdD1)s?$(;MLH@mx68mgTSLu~Ut$wfzsZC55b$8B9sivncwp{B1|HcO zivRfV_&O6FYE;_CFYtJa2amycJjQmibzXmhz%c>2B?cU{2IiDy z%Eunx@~PydS||2^1uPZr<=UN(Oo`&QDeTlE9{Ot6VpZv z%o9`MFL5AI-Mb15r@WSLK+d#vu!gdAFo$v;*iWhuOQpTtP8NjFO6hKx+iA#xY>oFN zvLIM+!@vNSIuR_xhqE~ERuw^|sC&{m(^j{L2tJrQVc?Uk@qympvxW_yl>(pB)fo_u z?YQRxLFbE*yV*d8*1+8R1v-*PJDOatlUHC4RTbUj%GOAg-CzSJgO0Hg;Da-j#<_A; zF6KnB?Nty&8HY8OXsQ-YCkDt!r+MrWV5W@t*C1!wBK8`ZAnC$Y@G?AC86}^E+${#3 zXbsGvb1$APgI%0-I><(+j@`WI$l&h#)iruEReR7rB?_jn42^}$<|1hw1#Iq4+2?u4ne1b)p#<6;Qlz3a0ISj+-}oH7V{Q7zYmi%R(8gXv2)7X7V_M$~ zX4z;nBtV+V9mdZB4y#gBFgIbKlC7b1BUV&C!GH>wNxZ|X&d_OLeXOO!tV;v<1)!)h zfIkU2bDY_0^k7Q=c{ZrJ@$S9^P$*re1vyjKp*1kqBVFh748S1cvntXLGIg5OJl)99 zX)#*0WtXGXx_drHi>d)vDbxLF$lYX!Us?lmQnwWE>8QQV#LrCjU}5fd;|5R57NGup_{C$`SSm zVs;A95} z&xZ*xnA>9jldYlLgRH=OhXEMg!SpsmkHuhGJ{8uc06_8I2q}7l!2NKq} zJ=N$jcaIoI;x(({jjZxLM`72kTP@~HPPai@HKH(QZl=lBP`1H4oFnRg7{~xq2cs1J zcZMDdir%6z4i%7;b=Cj@73(;VGg${~DAvK;rt4&#yP&ON9n6`mldYjxXUUC>TF349 z@OfU6;ANeq3_TXCvrJIr*l*Km}gDxrd?0f^Sxw^O9pF?Ig}Fpr7KLLy$8$2Wu#;4|8h;=Ojx@=((b= zpjzL4XfmA6c9=8Q_Odn5021cSgPaU=m<|5HMp$VF%3cBr<{p7h*4NvzHPS7g0V^nv zF@VCW@$F&gu^2^zXwtOVIerfiP7NQ-p(dx35zExham=Zms(c2e%ZA9Zs9Zf)@lgHE z-@scSuwZV~fCbAGESO8}8Ac6nnh};-BP_?uKWRYLw1El@5vw;ADITH{pQ-ENu7l8Z< z10WukbK74s&o-vN?s&sr?ceL~^bhza{ImW!*zfs_|AOE0f64!n{}umh{x|%;@&C#H ze(e7ahQZ$uti}YFeS)o+;0~Q&KPJa9c>u&BGV5&WU0 z7;I@`kY^MV4|BlLs}H`~c@pPdY{ z81z;<8RYMS2K8hyuu0GxGJJcU$q;-K9@q%}4$|hPnZ{(LTy)IP1iPK69)cNgcEZ`? zoSk@3qk(>4zZ>+o+0mE9AI(Ew{%)c#i*M~2`rl(j|8X6Cw6~(&&IsADik_E396$tr zcAjx8zothEM=jLS!U=Rkto!Z6lEttXXdf!_cavDM7zJH#hRT0263fwv1)4q3DtC-z zq84<6{U$s1viN~K?B(w!_Ocj`)%FPcn-|RQY@v&(?FsfGicS&?0%TLwyXeSuw4`*T z?w6xcU1X7X3EhxltDO|G7`0&GIWFZ zMmyrN7=ks}5tqN4h|A*M%@XSv5g(O^s~8uO@As!cFS0B@lq0s53MFsK3m>{rE0^|;5HV;3svtDTVT&n2gT)37d{GR? zU+gryxzSXj9-;C&KtYhO*M!MXhpxo_1I6F_rS=$jRJ=O&697%N2 zE{S9@#OSc2-M$&@Annm8_&6il4}lI$BAb9zaxc2&x`SF+A(@2)*yOOTA)9C{5JHt4 z*)(n+16~K&l+#QWE7^2CQ`Twm?iGf(l*PO97^QtPqBoY2mRwU-osm(lZ_3g`mJpuE z8L4ON1dzoL&BQ)Z+czU4m6ZTL#Yli{D*8)0M%CbzhhiT~BgD|*DUUFr6N0LfIdpj3 zP6t^Gr|a!>Xy2@Nboe<&I^3y12Rf+CzVMuGKxx5Px?%h$?0`>*Vu-h42Rx^lELQqE zxSClR|8tChUrxi?Xd3ejM1w5^wQ_YNRmf0dgH=k{RRu9xw~%5*CD7zV+*ce5xDUX2 z<{1aQbZ7_z3AJWK_a?z)x*_P3c7n>{(|H7K-walnmCnIR(AOCWdRU7?qmeY;Q;no3 zOhu>klm3rmm@!c@lJWwfP8X4F2+fA=7ON|0w%(Zjiya1G=Vqz*l$B;*VWio%tYPFh zd8U!$%p!U4xE!k)a-Q#l&&yMpqAb%DIVC_nC9G%%33+GEdBVoD#56!#{c_bs{#@^Df*y4yJ^e1gIoDBV)m zB{s}4oH9|@#(qq=$nX{ju8}mwwVZ^fW1bYX#ro&L#!SV#G@?9f88=VL8GW%EJ0DGw z^~J7TOHcX6ky8!f!R*|T{M;A21&i2C(R|nLarva#(fB`dItjgcBdiL|QG2_-NvLgv zK%&OyI2NlyUpKdHy&;No*TQ5ob1h63qt8RA@yfAi#1$lc%W5r$w)6-lsU<3<6gB<}f;p(H7pY|#|EA<_jqk-jL3A<&|oNIA`9 zF@#aejAuO#2%$lcUo8R+BYN11>fZy%>Tj2J#AO{aq_(bJ%yp2zXeXL1hT~y& zqP1^EmO@yGb_XNT+79x$E;+p-a9(LRUtbl)5OB?ob51i^tfV~n5?FEG$cXdj6}Ma6 zKBrQvynwL?M3yd$fNdma=EuCKQK=vB8`bl;tp)xs>eCeydwn#f#xKu!55vZQIvHie z9j`_BBqBRUC%$5*yex*ZhIY!gZ$_3%SSi1qk@Am%qf?Lv^1#S?i{YTt~QW2{vAF-9u=XcP;1l)!VC z14N@Ojt>hmURb~thGb=qRVIFj4~aAP!yys5SyOygvdXq4W=pHeSe|l5BnEYuznndL z_Bfkh$tAwZ6$342`5xV`$r9UBAV=jgnv=HL?QFQJ;X3OS3I!)RU!IwPgIDXGvZ$eQ zC>cY>fpLXSF;i`wHR|Pc2wB4HOZM%$H+epo$mpD4w8PL9sgL8WzjA33Uzb5cG?2lr zOPgQ?WPYU(>0i7Yol2#SL4b*<%9;3H5TeT5EAUBCoU5*OZkA3ctON8M-vDjSv?Ha? z#qt6o58{rS%%9<>T>FmF)`(h6D4AoGza&UzTIDTKoNJZmG?T@>v&!p?lzfGGDK615 zGSt?j2g(%KXp!L%&^2IZIIa=1I>{FB!F%Sn7N;|rMYqpPWH^ag>J?DF3af0oJT2!o zcV-681;@9vk6qdd(^qDW6B9yeAk3G2LF8`wbnQ}uEy+oX{^adW$Ls$6$vcq)@m~<= zgo(^NWh(F+;vIJF^#_hghxE}lEOqCK*e7ADWijB%_s=?BCGwo>9s6*^Ff;OP-OxKr zzRYpv)^}^;x$O)|xFLUsNK)fHK0O&&I5675g_GG(m=W~J=VhkFTqOuwnD)Z#=v)}7 z9LZzY&c==1dYVkLsrQU!TcaEq6*~?R6cY~4-sJ{0bx|XS+Eqd}-EKlvWDdqRd5+zo zst>z&>wqP%%dn22QIQXKv@N3Tms1pvCCV4;c7yNntx%qMa&TZE%Mw7|c!67jnd}wP zx3h^xnd|2sI=YYeXF<=YUf!#`@X}>r86%nTf4#!Kujuu?G}P5cWP)XwYf|Ao;iVz{ z?JIiPQe7t_udnE(i(YC!{Hb(dH1QGJPhuphLZ?|X&ksS^79ZAE^b$z%?>tx#eMK(; zw3FthiFDLgJ5|pyDRf0Ih*-4isEbbv;nl*JGl-`=nG&EUc?wUO(2bUSjbCcnhz(`?%I1mS&R ztaidD7(eKuIVpgNkn7Kg=NrM`4-wk|r|*KAj`pSpVeI25UesVe|MVw}=GF9)obyit z1S6CPIok9C0z4LT!rY(RPIRrIZJJ6nbawitQQE47-*#bXeFOKJ!^gh zE@J6NT&!0L6MH*nE7TET*P1719}&g5Cue0d^W>~7#;D`wZf^Y|V$!i3N5E0h1Je;8 zi}&Xl-0ho@lXkz%X>k84qkXzd-RTUacs5&t<*zNr<@O|?cydd2g?_^wEYo4Aj zNcWy81iC~?GhVNo@8f5Hb#1rx3GIYLR&SEt)c7n!9LRtOcT)~$ZlfmSDn|ydvdiGr z9b~XrfxX)|Bd0X}lv5c072hamZjRTU z_K9l_l8eECwb5aCYT6Bx3t!{$j&4!b_(p%m_@)0K2Rqz-UdIb);uuHTO~|T4!*eCdV!uTWfP9 zTdoHuBPC}nud$D1r^8s5&CIbZi{U!)`5Vi3*~hXh&K=9`n~|~n9nP`*eMSy{Z;j=D z;rr4vo7LQ%H=L!7B8FSM%0Yu4#zh4TKkQD@i;l!EBK((r0g@_a=T2B#kn$}eKR5x1|T>d+v?^L zd^l5>iuaaGxRLQo|GdNe4mW6xG{VXcZf1uLz}OXCnDfA0VFr}yDt^l@`_MhqVd%BpI&jhjjqZ;l$%X0E^Smqqr$EI+ zPWVWzWZX`YgMPt|zAVnY5Lo_hqA!by>zNh(lZ@ym7XnjMcRQ01f!>21L*NE+;XI)p z251?b?YiRs&2|dN;@qHU@^_O0vbc8?m|\NK1()(DF+U!(siiRrBaKP-T^X%jh& zf3F>OS)3awOa5-cE{lma!OHjnBkWHR*afEA+NXE;`MoG%im43WidtpNo^t4G4dQE z+Uq3R37&Fhh84PaNs>;?i<9}fE7UjGQJ2NJsLS6?)Mat+P=B!ps3&+z)SoENDAYYR zZ`kkgU8#QNoz(*a;#v!*SaG9>EOEuw1uT@F8s9g5^Z;n%$xzQ(FoMDV!{$)J{XHGN z3ia9_@@23tzEaxfvB?$1Q|f@>i(GN9^?=x&Sqn*j zP^sjV;M%D;T433WlJ7Y?0a%@#K#zhH11{FYF%NY{vd+eM#T$Rt7s7C@dkV_5<3Xc2 zPX@5FFNe3s4xF%2tk**H8O_E_xOp@D;%s)ns{{u^e9x(c<#QN-BsmEp7rx?JW7;>w zOZk0Yl-wVLq!D}y#LkkmEB`S<4F9B`PCZfcMd|ED?u!EL?8p@?a2CH|A3Ug6r_UHghD%n|ul-Z^{eBWo2MYNUq18Vbdg-s!$M63WAc5 zy^gSdUtu*W#CKD1UvI0?$FlqqPe~x%)TN7w@QXXwzpt5u^;j-PfefH z-NjkC;!;c*{d2Z%eU18zjL(!zrVk=^v9{hnXIpw~4rlH9ai;olruuQF5`AXA zqFASM2?`~b#FES2)*=oj*lP6yie|2wML_XizUC}`XV;*U*C-V@8!cZd#zcJnG$-OboUAEPCrU(>V5C-obAwT z%U^fA5xfNpE5Wxg`7S2^iOC`es1vNjWGyCJFnIuz{g|A<oK_xlYN*R$K)&~J|+!Jp2g%-ka!cV O_&k$)t#z%{DEu#|jzl#8 diff --git a/changelog.md b/changelog.md index 9dd8e1e..ce83c42 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,21 @@ # Changelog +## 2025-08-18 - 4.2.0 - feat(classes.smartarchive) +Support URL streams, recursive archive unpacking and filesystem export; improve ZIP/GZIP/BZIP2 robustness; CI and package metadata updates + +- Add exportToFs(targetDir, fileName?) to write extracted StreamFile objects to the filesystem (ensures directories, logs relative paths, waits for write completion). +- Implement exportToStreamOfStreamFiles with recursive unpacking pipeline that handles application/x-tar (tar-stream Extract), application/zip (fflate Unzip), nested archives and StreamIntake for StreamFile results. +- Enhance getArchiveStream() to support URL/web streams (SmartRequest) and return Node Readable streams for remote archives. +- Make ZIP decompression more robust: accept ArrayBuffer-like chunks, coerce to Buffer before pushing to fflate.Unzip, and ensure SmartDuplex handling of results. +- Fixes and improvements to bzip2/gzip/tar tool implementations (various bug/formatting fixes, safer CRC and stream handling). +- Update CI workflows to use new registry image and adjust npmci install path; minor .gitignore additions. +- Package metadata tweaks: bugs URL and homepage updated, packageManager/pnpm fields adjusted. +- Documentation/readme expanded and polished with quick start, examples and API reference updates. +- Small test and plugin export cleanups (formatting and trailing commas removed/added). +- TypeScript/formatting fixes across many files (consistent casing, trailing commas, typings, tsconfig additions). + ## 2025-08-18 - 4.1.0 - feat(classes.smartarchive) + Support URL web streams, add recursive archive unpacking and filesystem export, and improve ZIP decompression robustness - ts/classes.smartarchive.ts: add exportToFs(targetDir, fileName?) to write extracted StreamFile objects to the filesystem (ensures directories, logs relative paths, waits for write completion). @@ -9,6 +24,7 @@ Support URL web streams, add recursive archive unpacking and filesystem export, - ts/classes.ziptools.ts: make ZIP decompression writeFunction more robust β€” accept non-Buffer chunks, coerce to Buffer before pushing to fflate.Unzip, and loosen the writeFunction typing to handle incoming ArrayBuffer-like data. ## 2024-10-13 - 4.0.39 - fix(core) + Fix dependencies and update documentation. - Ensure package uses the latest dependencies @@ -16,6 +32,7 @@ Fix dependencies and update documentation. - Updated readme with advanced usage examples ## 2024-10-13 - 4.0.38 - fix(dependencies) + Update dependencies to latest versions - Updated @push.rocks/smartfile to version 11.0.21 @@ -28,11 +45,13 @@ Update dependencies to latest versions - Updated @push.rocks/tapbundle to version 5.3.0 ## 2024-06-08 - 4.0.24 to 4.0.37 - Fixes and Updates + Core updates and bug fixes were implemented in versions 4.0.24 through 4.0.37. - Repeated core updates and fixes applied consistently across multiple versions. ## 2024-06-06 - 4.0.22 to 4.0.23 - Descriptions and Fixes Updates + Efforts to update documentation and core features. - "update description" in 4.0.22 @@ -40,24 +59,28 @@ Efforts to update documentation and core features. - Ongoing core fixes. ## 2023-11-06 - 4.0.0 - Major Update with Breaking Changes + Introduction of significant updates and breaking changes. - Transition to new version 4.0.0 with core updates. - Break in compatibility due to major structural changes with core functionalities. ## 2023-07-11 - 3.0.6 - Organizational Changes + Structural reorganization and updates to the organization schema. - Switch to new organizational schema implemented. ## 2022-04-04 - 3.0.0 - Build Updates and Breaking Changes + Major build update introducing breaking changes. - Introduction of ESM structure with breaking changes. ## 2016-01-18 - 0.0.0 to 1.0.0 - Initial Development and Launch + Initial software development and establishment of core features. - Project set-up including Travis CI integration. - Launch of the first full version with code restructuring. -- Added callback support. \ No newline at end of file +- Added callback support. diff --git a/npmextra.json b/npmextra.json index c50eafe..e68e915 100644 --- a/npmextra.json +++ b/npmextra.json @@ -30,4 +30,4 @@ "tsdoc": { "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n" } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 1518c50..b370bcf 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,9 @@ "author": "Lossless GmbH", "license": "MIT", "bugs": { - "url": "https://github.com/pushrocks/smartarchive/issues" + "url": "https://code.foss.global/push.rocks/smartarchive/issues" }, - "homepage": "https://code.foss.global/push.rocks/smartarchive", + "homepage": "https://code.foss.global/push.rocks/smartarchive#readme", "dependencies": { "@push.rocks/smartdelay": "^3.0.5", "@push.rocks/smartfile": "^11.2.7", @@ -70,5 +70,8 @@ "data analysis", "file stream" ], - "packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748" + "packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748", + "pnpm": { + "overrides": {} + } } diff --git a/readme.md b/readme.md index b72483e..b4f670e 100644 --- a/readme.md +++ b/readme.md @@ -1,266 +1,333 @@ -# @push.rocks/smartarchive +# @push.rocks/smartarchive πŸ“¦ -`@push.rocks/smartarchive` is a powerful library designed for managing archive files. It provides utilities for compressing and decompressing data in various formats such as zip, tar, gzip, and bzip2. This library aims to simplify the process of handling archive files, making it an ideal choice for projects that require manipulation of archived data. +**Powerful archive manipulation for modern Node.js applications** -## Install +`@push.rocks/smartarchive` is a versatile library for handling archive files with a focus on developer experience. Work with **zip**, **tar**, **gzip**, and **bzip2** formats through a unified, streaming-optimized API. -To install `@push.rocks/smartarchive`, you can either use npm or yarn. Run one of the following commands in your project directory: +## Features πŸš€ -```shell -npm install @push.rocks/smartarchive --save -``` +- πŸ“ **Multi-format support** - Handle `.zip`, `.tar`, `.tar.gz`, `.tgz`, and `.bz2` archives +- 🌊 **Streaming-first architecture** - Process large archives without memory constraints +- πŸ”„ **Unified API** - Consistent interface across different archive formats +- 🎯 **Smart detection** - Automatically identifies archive types +- ⚑ **High performance** - Optimized for speed with parallel processing where possible +- πŸ”§ **Flexible I/O** - Work with files, URLs, and streams seamlessly +- πŸ“Š **Archive analysis** - Inspect contents without extraction +- πŸ› οΈ **Modern TypeScript** - Full type safety and excellent IDE support -or if you prefer yarn: +## Installation πŸ“₯ -```shell +```bash +# Using npm +npm install @push.rocks/smartarchive + +# Using pnpm (recommended) +pnpm add @push.rocks/smartarchive + +# Using yarn yarn add @push.rocks/smartarchive ``` -This will add `@push.rocks/smartarchive` to your project's dependencies. +## Quick Start 🎯 -## Usage -`@push.rocks/smartarchive` provides an easy-to-use API for extracting, creating, and analyzing archive files. Below, we'll cover how to get started and explore various features of the module. - -### Importing SmartArchive - -First, import `SmartArchive` from `@push.rocks/smartarchive` using ESM syntax: - -```typescript -import { SmartArchive } from '@push.rocks/smartarchive'; -``` - -### Extracting Archive Files - -You can extract archive files from different sources using `SmartArchive.fromArchiveUrl`, `SmartArchive.fromArchiveFile`, and `SmartArchive.fromArchiveStream`. Here's an example of extracting an archive from a URL: +### Extract an archive from URL ```typescript import { SmartArchive } from '@push.rocks/smartarchive'; -async function extractArchiveFromURL() { - const url = 'https://example.com/archive.zip'; - const targetDir = '/path/to/extract'; - - const archive = await SmartArchive.fromArchiveUrl(url); - await archive.exportToFs(targetDir); - - console.log('Archive extracted successfully.'); -} - -extractArchiveFromURL(); +// Extract a .tar.gz archive from a URL directly to the filesystem +const archive = await SmartArchive.fromArchiveUrl( + 'https://github.com/some/repo/archive/main.tar.gz' +); +await archive.exportToFs('./extracted'); ``` -### Extracting an Archive from a File - -Similarly, you can extract an archive from a local file: +### Process archive as a stream ```typescript import { SmartArchive } from '@push.rocks/smartarchive'; -async function extractArchiveFromFile() { - const filePath = '/path/to/archive.zip'; - const targetDir = '/path/to/extract'; +// Stream-based processing for memory efficiency +const archive = await SmartArchive.fromArchiveFile('./large-archive.zip'); +const streamOfFiles = await archive.exportToStreamOfStreamFiles(); - const archive = await SmartArchive.fromArchiveFile(filePath); - await archive.exportToFs(targetDir); - - console.log('Archive extracted successfully.'); -} - -extractArchiveFromFile(); +// Process each file in the archive +streamOfFiles.on('data', (fileStream) => { + console.log(`Processing ${fileStream.path}`); + // Handle individual file stream +}); ``` -### Stream-Based Extraction +## Core Concepts πŸ’‘ -For larger files, you might prefer a streaming approach to prevent high memory consumption. Here’s an example: +### Archive Sources -```typescript -import { SmartArchive } from '@push.rocks/smartarchive'; -import { createReadStream } from 'fs'; +`SmartArchive` accepts archives from three sources: -async function extractArchiveUsingStream() { - const archiveStream = createReadStream('/path/to/archive.zip'); - const archive = await SmartArchive.fromArchiveStream(archiveStream); - const extractionStream = await archive.exportToStreamOfStreamFiles(); - - extractionStream.pipe(createWriteStream('/path/to/destination')); -} +1. **URL** - Download and process archives from the web +2. **File** - Load archives from the local filesystem +3. **Stream** - Process archives from any Node.js stream -extractArchiveUsingStream(); -``` +### Export Destinations -### Analyzing Archive Files +Extract archives to multiple destinations: -Sometimes, you may need to inspect the contents of an archive before extracting it. The following example shows how to analyze an archive: +1. **Filesystem** - Extract directly to a directory +2. **Stream of files** - Process files individually as streams +3. **Archive stream** - Re-stream as different format + +## Usage Examples πŸ”¨ + +### Working with ZIP files ```typescript import { SmartArchive } from '@push.rocks/smartarchive'; -async function analyzeArchive() { - const filePath = '/path/to/archive.zip'; - - const archive = await SmartArchive.fromArchiveFile(filePath); - const analysisResult = await archive.analyzeContent(); - - console.log(analysisResult); // Outputs details about the archive content -} +// Extract a ZIP file +const zipArchive = await SmartArchive.fromArchiveFile('./archive.zip'); +await zipArchive.exportToFs('./output'); -analyzeArchive(); +// Stream ZIP contents for processing +const fileStream = await zipArchive.exportToStreamOfStreamFiles(); +fileStream.on('data', (file) => { + if (file.path.endsWith('.json')) { + // Process JSON files from the archive + file.pipe(jsonProcessor); + } +}); ``` -### Creating Archive Files - -Creating an archive file is straightforward. Here we demonstrate creating a tar.gz archive: - -```typescript -import { SmartArchive } from '@push.rocks/smartarchive'; - -async function createTarGzArchive() { - const archive = new SmartArchive(); - - // Add directories and files - archive.addedDirectories.push('/path/to/directory1'); - archive.addedFiles.push('/path/to/file1.txt'); - - // Export as tar.gz - const tarGzStream = await archive.exportToTarGzStream(); - - // Save to filesystem or handle as needed - tarGzStream.pipe(createWriteStream('/path/to/destination.tar.gz')); -} - -createTarGzArchive(); -``` - -### Stream Operations - -Here's an example of using `smartarchive`'s streaming capabilities: - -```typescript -import { createReadStream, createWriteStream } from 'fs'; -import { SmartArchive } from '@push.rocks/smartarchive'; - -async function extractArchiveUsingStreams() { - const archiveStream = createReadStream('/path/to/archive.zip'); - const archive = await SmartArchive.fromArchiveStream(archiveStream); - const extractionStream = await archive.exportToStreamOfStreamFiles(); - - extractionStream.pipe(createWriteStream('/path/to/extracted')); -} - -extractArchiveUsingStreams(); -``` - -### Advanced Decompression Usage - -`smartarchive` supports multiple compression formats. It also provides detailed control over the decompression processes: - -- For ZIP files, `ZipTools` handles decompression using the `fflate` library. -- For TAR files, `TarTools` uses `tar-stream`. -- For GZIP files, `GzipTools` provides a `CompressGunzipTransform` and `DecompressGunzipTransform`. -- For BZIP2 files, `Bzip2Tools` utilizes custom streaming decompression. - -Example: Working with a GZIP-compressed archive: - -```typescript -import { createReadStream, createWriteStream } from 'fs'; -import { SmartArchive } from '@push.rocks/smartarchive'; - -async function decompressGzipArchive() { - const filePath = '/path/to/archive.gz'; - const targetDir = '/path/to/extract'; - - const archive = await SmartArchive.fromArchiveFile(filePath); - await archive.exportToFs(targetDir); - - console.log('GZIP archive decompressed successfully.'); -} - -decompressGzipArchive(); -``` - -### Advancing with Custom Decompression Streams - -You can inject custom decompression streams where needed: - -```typescript -import { createReadStream, createWriteStream } from 'fs'; -import { SmartArchive, GzipTools } from '@push.rocks/smartarchive'; - -async function customDecompression() { - const filePath = '/path/to/archive.gz'; - const targetDir = '/path/to/extract'; - - const archive = await SmartArchive.fromArchiveFile(filePath); - const gzipTools = new GzipTools(); - const decompressionStream = gzipTools.getDecompressionStream(); - - const archiveStream = await archive.getArchiveStream(); - archiveStream.pipe(decompressionStream).pipe(createWriteStream(targetDir)); - - console.log('Custom GZIP decompression successful.'); -} - -customDecompression(); -``` - -### Custom Pack and Unpack Tar - -When dealing with tar archives, you may need to perform custom packing and unpacking: +### Working with TAR archives ```typescript import { SmartArchive, TarTools } from '@push.rocks/smartarchive'; -import { createWriteStream } from 'fs'; -async function customTarOperations() { - const tarTools = new TarTools(); +// Extract a .tar.gz file +const tarGzArchive = await SmartArchive.fromArchiveFile('./archive.tar.gz'); +await tarGzArchive.exportToFs('./extracted'); - // Packing a directory into a tar stream - const packStream = await tarTools.packDirectory('/path/to/directory'); - packStream.pipe(createWriteStream('/path/to/archive.tar')); - - // Extracting files from a tar stream - const extractStream = tarTools.getDecompressionStream(); - createReadStream('/path/to/archive.tar').pipe(extractStream).on('entry', (header, stream, next) => { - const writeStream = createWriteStream(`/path/to/extract/${header.name}`); - stream.pipe(writeStream); - stream.on('end', next); - }); -} - -customTarOperations(); +// Create a TAR archive (using TarTools directly) +const tarTools = new TarTools(); +const packStream = await tarTools.packDirectory('./source-directory'); +packStream.pipe(createWriteStream('./output.tar')); ``` -### Extract and Analyze All-in-One - -To extract and simultaneously analyze archive content: +### Extracting from URLs ```typescript -import { createReadStream, createWriteStream } from 'fs'; import { SmartArchive } from '@push.rocks/smartarchive'; -async function extractAndAnalyze() { - const filePath = '/path/to/archive.zip'; - const targetDir = '/path/to/extract'; +// Download and extract in one operation +const remoteArchive = await SmartArchive.fromArchiveUrl( + 'https://example.com/data.tar.gz' +); - const archive = await SmartArchive.fromArchiveFile(filePath); - const analyzedStream = archive.archiveAnalyzer.getAnalyzedStream(); - const extractionStream = await archive.exportToStreamOfStreamFiles(); +// Extract to filesystem +await remoteArchive.exportToFs('./local-dir'); - analyzedStream.pipe(extractionStream).pipe(createWriteStream(targetDir)); - - analyzedStream.on('data', (chunk) => { - console.log(JSON.stringify(chunk, null, 2)); - }); -} - -extractAndAnalyze(); +// Or process as stream +const stream = await remoteArchive.exportToStreamOfStreamFiles(); ``` -### Final Words +### Analyzing archive contents -These examples demonstrate various use cases for `@push.rocks/smartarchive`. Depending on your specific project requirements, you can adapt these examples to suit your needs. Always refer to the latest documentation for the most current information and methods available in `@push.rocks/smartarchive`. +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; -For more information and API references, check the official [`@push.rocks/smartarchive` GitHub repository](https://code.foss.global/push.rocks/smartarchive). +// Analyze without extracting +const archive = await SmartArchive.fromArchiveFile('./archive.zip'); +const analyzer = archive.archiveAnalyzer; + +// Use the analyzer to inspect contents +// (exact implementation depends on analyzer methods) +``` + +### Working with GZIP files + +```typescript +import { SmartArchive, GzipTools } from '@push.rocks/smartarchive'; + +// Decompress a .gz file +const gzipArchive = await SmartArchive.fromArchiveFile('./data.json.gz'); +await gzipArchive.exportToFs('./decompressed', 'data.json'); + +// Use GzipTools directly for streaming +const gzipTools = new GzipTools(); +const decompressStream = gzipTools.getDecompressionStream(); + +createReadStream('./compressed.gz') + .pipe(decompressStream) + .pipe(createWriteStream('./decompressed')); +``` + +### Working with BZIP2 files + +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; + +// Handle .bz2 files +const bzipArchive = await SmartArchive.fromArchiveUrl( + 'https://example.com/data.bz2' +); +await bzipArchive.exportToFs('./extracted', 'data.txt'); +``` + +### Advanced streaming operations + +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; +import { pipeline } from 'stream/promises'; + +// Chain operations with streams +const archive = await SmartArchive.fromArchiveFile('./archive.tar.gz'); +const exportStream = await archive.exportToStreamOfStreamFiles(); + +// Process each file in the archive +await pipeline( + exportStream, + async function* (source) { + for await (const file of source) { + if (file.path.endsWith('.log')) { + // Process log files + yield processLogFile(file); + } + } + }, + createWriteStream('./processed-logs.txt') +); +``` + +### Creating archives (advanced) + +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; +import { TarTools } from '@push.rocks/smartarchive'; + +// Using SmartArchive to create an archive +const archive = new SmartArchive(); + +// Add content to the archive +archive.addedDirectories.push('./src'); +archive.addedFiles.push('./readme.md'); +archive.addedFiles.push('./package.json'); + +// Export as TAR.GZ +const tarGzStream = await archive.exportToTarGzStream(); +tarGzStream.pipe(createWriteStream('./output.tar.gz')); +``` + +### Extract and transform + +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; +import { Transform } from 'stream'; + +// Extract and transform files in one pipeline +const archive = await SmartArchive.fromArchiveUrl( + 'https://example.com/source-code.tar.gz' +); + +const extractStream = await archive.exportToStreamOfStreamFiles(); + +// Transform TypeScript to JavaScript during extraction +extractStream.on('data', (fileStream) => { + if (fileStream.path.endsWith('.ts')) { + fileStream + .pipe(typescriptTranspiler()) + .pipe(createWriteStream(fileStream.path.replace('.ts', '.js'))); + } else { + fileStream.pipe(createWriteStream(fileStream.path)); + } +}); +``` + +## API Reference πŸ“š + +### SmartArchive Class + +#### Static Methods + +- `SmartArchive.fromArchiveUrl(url: string)` - Create from URL +- `SmartArchive.fromArchiveFile(path: string)` - Create from file +- `SmartArchive.fromArchiveStream(stream: NodeJS.ReadableStream)` - Create from stream + +#### Instance Methods + +- `exportToFs(targetDir: string, fileName?: string)` - Extract to filesystem +- `exportToStreamOfStreamFiles()` - Get a stream of file streams +- `exportToTarGzStream()` - Export as TAR.GZ stream +- `getArchiveStream()` - Get the raw archive stream + +#### Properties + +- `archiveAnalyzer` - Analyze archive contents +- `tarTools` - TAR-specific operations +- `zipTools` - ZIP-specific operations +- `gzipTools` - GZIP-specific operations +- `bzip2Tools` - BZIP2-specific operations + +### Specialized Tools + +Each tool class provides format-specific operations: + +- **TarTools** - Pack/unpack TAR archives +- **ZipTools** - Handle ZIP compression +- **GzipTools** - GZIP compression/decompression +- **Bzip2Tools** - BZIP2 operations + +## Performance Tips 🏎️ + +1. **Use streaming for large files** - Avoid loading entire archives into memory +2. **Process files in parallel** - Utilize stream operations for concurrent processing +3. **Choose the right format** - TAR.GZ for Unix systems, ZIP for cross-platform compatibility +4. **Enable compression wisely** - Balance between file size and CPU usage + +## Error Handling πŸ›‘οΈ + +```typescript +import { SmartArchive } from '@push.rocks/smartarchive'; + +try { + const archive = await SmartArchive.fromArchiveUrl('https://example.com/file.zip'); + await archive.exportToFs('./output'); +} catch (error) { + if (error.code === 'ENOENT') { + console.error('Archive file not found'); + } else if (error.code === 'EACCES') { + console.error('Permission denied'); + } else { + console.error('Archive extraction failed:', error.message); + } +} +``` + +## Real-World Use Cases 🌍 + +### Backup System +```typescript +// Automated backup extraction +const backup = await SmartArchive.fromArchiveFile('./backup.tar.gz'); +await backup.exportToFs('/restore/location'); +``` + +### CI/CD Pipeline +```typescript +// Download and extract build artifacts +const artifacts = await SmartArchive.fromArchiveUrl( + `${CI_SERVER}/artifacts/build-${BUILD_ID}.zip` +); +await artifacts.exportToFs('./dist'); +``` + +### Data Processing +```typescript +// Process compressed datasets +const dataset = await SmartArchive.fromArchiveUrl( + 'https://data.source/dataset.tar.bz2' +); +const files = await dataset.exportToStreamOfStreamFiles(); +// Process each file in the dataset +``` ## License and Legal Information @@ -279,4 +346,4 @@ Registered at District court Bremen HRB 35230 HB, Germany For any legal inquiries or if you require further information, please contact us via email at hello@task.vc. -By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works. +By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works. \ No newline at end of file diff --git a/test/plugins.ts b/test/plugins.ts index 703e3b9..af709f1 100644 --- a/test/plugins.ts +++ b/test/plugins.ts @@ -4,10 +4,4 @@ import * as smartfile from '@push.rocks/smartfile'; import * as smartrequest from '@push.rocks/smartrequest'; import * as smartstream from '@push.rocks/smartstream'; -export { - path, - smartpath, - smartfile, - smartrequest, - smartstream, -} +export { path, smartpath, smartfile, smartrequest, smartstream }; diff --git a/test/test.ts b/test/test.ts index 9012d10..43baa43 100644 --- a/test/test.ts +++ b/test/test.ts @@ -5,11 +5,11 @@ import * as plugins from './plugins.js'; const testPaths = { nogitDir: plugins.path.join( plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), - '../.nogit/' + '../.nogit/', ), remoteDir: plugins.path.join( plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), - '../.nogit/remote' + '../.nogit/remote', ), }; @@ -21,29 +21,32 @@ tap.preTask('should prepare .nogit dir', async () => { tap.preTask('should prepare downloads', async (tools) => { const response = await plugins.smartrequest.SmartRequest.create() - .url('https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz') - .get(); + .url( + 'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz', + ) + .get(); const downloadedFile: Buffer = Buffer.from(await response.arrayBuffer()); await plugins.smartfile.memory.toFs( downloadedFile, - plugins.path.join(testPaths.nogitDir, 'test.tgz') + plugins.path.join(testPaths.nogitDir, 'test.tgz'), ); }); tap.test('should extract existing files on disk', async () => { const testSmartarchive = await smartarchive.SmartArchive.fromArchiveUrl( - 'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz' + 'https://verdaccio.lossless.digital/@pushrocks%2fwebsetup/-/websetup-2.0.14.tgz', ); await testSmartarchive.exportToFs(testPaths.nogitDir); }); tap.skip.test('should extract a b2zip', async () => { - const dataUrl = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; + const dataUrl = + 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'; const testArchive = await smartarchive.SmartArchive.fromArchiveUrl(dataUrl); - await testArchive.exportToFs( - plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'), - 'data.jsonl', - ); -}) + await testArchive.exportToFs( + plugins.path.join(testPaths.nogitDir, 'de_companies_ocdata.jsonl'), + 'data.jsonl', + ); +}); await tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index f5a4608..4e5e428 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartarchive', - version: '4.1.0', + version: '4.2.0', description: 'A library for working with archive files, providing utilities for compressing and decompressing data.' } diff --git a/ts/bzip2/bititerator.ts b/ts/bzip2/bititerator.ts index 3c6b335..5652992 100644 --- a/ts/bzip2/bititerator.ts +++ b/ts/bzip2/bititerator.ts @@ -1,41 +1,44 @@ -var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF]; +var BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff]; // returns a function that reads bits. // takes a buffer iterator as input export function bitIterator(nextBuffer: () => Buffer) { - var bit = 0, byte = 0; - var bytes = nextBuffer(); - var f = function(n) { - if (n === null && bit != 0) { // align to byte boundary - bit = 0 - byte++; - return; - } - var result = 0; - while(n > 0) { - if (byte >= bytes.length) { - byte = 0; - bytes = nextBuffer(); - } - var left = 8 - bit; - if (bit === 0 && n > 0) - // @ts-ignore - f.bytesRead++; - if (n >= left) { - result <<= left; - result |= (BITMASK[left] & bytes[byte++]); - bit = 0; - n -= left; - } else { - result <<= n; - result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); - bit += n; - n = 0; - } - } - return result; - }; - // @ts-ignore - f.bytesRead = 0; - return f; -}; \ No newline at end of file + var bit = 0, + byte = 0; + var bytes = nextBuffer(); + var f = function (n) { + if (n === null && bit != 0) { + // align to byte boundary + bit = 0; + byte++; + return; + } + var result = 0; + while (n > 0) { + if (byte >= bytes.length) { + byte = 0; + bytes = nextBuffer(); + } + var left = 8 - bit; + if (bit === 0 && n > 0) + // @ts-ignore + f.bytesRead++; + if (n >= left) { + result <<= left; + result |= BITMASK[left] & bytes[byte++]; + bit = 0; + n -= left; + } else { + result <<= n; + result |= + (bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit); + bit += n; + n = 0; + } + } + return result; + }; + // @ts-ignore + f.bytesRead = 0; + return f; +} diff --git a/ts/bzip2/bzip2.ts b/ts/bzip2/bzip2.ts index 245ef23..9a27bea 100644 --- a/ts/bzip2/bzip2.ts +++ b/ts/bzip2/bzip2.ts @@ -1,7 +1,7 @@ export class Bzip2Error extends Error { public name: string = 'Bzip2Error'; public message: string; - public stack = (new Error()).stack; + public stack = new Error().stack; constructor(messageArg: string) { super(); @@ -10,326 +10,322 @@ export class Bzip2Error extends Error { } var messageArg = { - Error: function(message) {throw new Bzip2Error(message);} + Error: function (message) { + throw new Bzip2Error(message); + }, }; export class Bzip2 { public Bzip2Error = Bzip2Error; - public crcTable = - [ - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, - 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, - 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, - 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, - 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, - 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, - 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, - 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, - 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, - 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, - 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, - 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, - 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, - 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, - 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, - 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, - 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, - 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, - 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, - 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, - 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, - 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, - 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, - 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, - 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, - 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, - 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, - 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, - 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, - 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, - 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, - 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, - 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, - 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, - 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, - 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, - 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, - 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, - 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, - 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, - 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, - 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, - 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, - 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, - 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, - 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, - 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, - 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, - 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, - 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, - 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, - 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, - 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, - 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + public crcTable = [ + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b, + 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, + 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3, + 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, + 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, + 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0, + 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, + 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, + 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, + 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, + 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34, + 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1, + 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5, + 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9, + 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, + 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, + 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, + 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e, + 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a, + 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, + 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, + 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4, ]; - array = function(bytes) { - var bit = 0, byte = 0; - var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; - return function(n) { - var result = 0; - while(n > 0) { - var left = 8 - bit; - if (n >= left) { - result <<= left; - result |= (BITMASK[left] & bytes[byte++]); - bit = 0; - n -= left; - } else { - result <<= n; - result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); - bit += n; - n = 0; - } + array = function (bytes) { + var bit = 0, + byte = 0; + var BITMASK = [0, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff]; + return function (n) { + var result = 0; + while (n > 0) { + var left = 8 - bit; + if (n >= left) { + result <<= left; + result |= BITMASK[left] & bytes[byte++]; + bit = 0; + n -= left; + } else { + result <<= n; + result |= + (bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit); + bit += n; + n = 0; } - return result; - } - } + } + return result; + }; + }; - simple = function(srcbuffer, stream) { + simple = function (srcbuffer, stream) { var bits = this.array(srcbuffer); var size = this.header(bits); var ret = false; var bufsize = 100000 * size; var buf = new Int32Array(bufsize); - - do { - ret = this.decompress(bits, stream, buf, bufsize); - } while(!ret); - } - header = function(bits) { + do { + ret = this.decompress(bits, stream, buf, bufsize); + } while (!ret); + }; + + header = function (bits) { this.byteCount = new Int32Array(256); this.symToByte = new Uint8Array(256); this.mtfSymbol = new Int32Array(256); this.selectors = new Uint8Array(0x8000); - - if (bits(8*3) != 4348520) messageArg.Error("No magic number found"); - + + if (bits(8 * 3) != 4348520) messageArg.Error('No magic number found'); + var i = bits(8) - 48; - if (i < 1 || i > 9) messageArg.Error("Not a BZIP archive"); + if (i < 1 || i > 9) messageArg.Error('Not a BZIP archive'); return i; }; - decompress = function(bits, stream, buf, bufsize, streamCRC) { + decompress = function (bits, stream, buf, bufsize, streamCRC) { var MAX_HUFCODE_BITS = 20; var MAX_SYMBOLS = 258; var SYMBOL_RUNA = 0; var SYMBOL_RUNB = 1; var GROUP_SIZE = 50; - var crc = 0 ^ (-1); - - for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); - if (h == "177245385090") { - var finalCRC = bits(32)|0; - if (finalCRC !== streamCRC) messageArg.Error("Error in bzip2: crc32 do not match"); + var crc = 0 ^ -1; + + for (var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); + if (h == '177245385090') { + var finalCRC = bits(32) | 0; + if (finalCRC !== streamCRC) + messageArg.Error('Error in bzip2: crc32 do not match'); // align stream to byte bits(null); return null; // reset streamCRC for next call } - if (h != "314159265359") messageArg.Error("eek not valid bzip data"); - var crcblock = bits(32)|0; // CRC code - if (bits(1)) messageArg.Error("unsupported obsolete version"); + if (h != '314159265359') messageArg.Error('eek not valid bzip data'); + var crcblock = bits(32) | 0; // CRC code + if (bits(1)) messageArg.Error('unsupported obsolete version'); var origPtr = bits(24); - if (origPtr > bufsize) messageArg.Error("Initial position larger than buffer size"); + if (origPtr > bufsize) + messageArg.Error('Initial position larger than buffer size'); var t = bits(16); var symTotal = 0; for (i = 0; i < 16; i++) { - if (t & (1 << (15 - i))) { - var k = bits(16); - for(j = 0; j < 16; j++) { - if (k & (1 << (15 - j))) { - this.symToByte[symTotal++] = (16 * i) + j; - } - } + if (t & (1 << (15 - i))) { + var k = bits(16); + for (j = 0; j < 16; j++) { + if (k & (1 << (15 - j))) { + this.symToByte[symTotal++] = 16 * i + j; + } } + } } - + var groupCount = bits(3); - if (groupCount < 2 || groupCount > 6) messageArg.Error("another error"); + if (groupCount < 2 || groupCount > 6) messageArg.Error('another error'); var nSelectors = bits(15); - if (nSelectors == 0) messageArg.Error("meh"); - for(var i = 0; i < groupCount; i++) this.mtfSymbol[i] = i; - - for(var i = 0; i < nSelectors; i++) { - for(var j = 0; bits(1); j++) if (j >= groupCount) messageArg.Error("whoops another error"); - var uc = this.mtfSymbol[j]; - for(var k: any = j-1; k>=0; k--) { - this.mtfSymbol[k+1] = this.mtfSymbol[k]; - } - this.mtfSymbol[0] = uc; - this.selectors[i] = uc; + if (nSelectors == 0) messageArg.Error('meh'); + for (var i = 0; i < groupCount; i++) this.mtfSymbol[i] = i; + + for (var i = 0; i < nSelectors; i++) { + for (var j = 0; bits(1); j++) + if (j >= groupCount) messageArg.Error('whoops another error'); + var uc = this.mtfSymbol[j]; + for (var k: any = j - 1; k >= 0; k--) { + this.mtfSymbol[k + 1] = this.mtfSymbol[k]; + } + this.mtfSymbol[0] = uc; + this.selectors[i] = uc; } - + var symCount = symTotal + 2; var groups = []; var length = new Uint8Array(MAX_SYMBOLS), - temp = new Uint16Array(MAX_HUFCODE_BITS+1); - + temp = new Uint16Array(MAX_HUFCODE_BITS + 1); + var hufGroup; - - for(var j = 0; j < groupCount; j++) { - t = bits(5); //lengths - for(var i = 0; i < symCount; i++) { - while(true){ - if (t < 1 || t > MAX_HUFCODE_BITS) messageArg.Error("I gave up a while ago on writing error messages"); - if (!bits(1)) break; - if (!bits(1)) t++; - else t--; - } - length[i] = t; + + for (var j = 0; j < groupCount; j++) { + t = bits(5); //lengths + for (var i = 0; i < symCount; i++) { + while (true) { + if (t < 1 || t > MAX_HUFCODE_BITS) + messageArg.Error('I gave up a while ago on writing error messages'); + if (!bits(1)) break; + if (!bits(1)) t++; + else t--; } - var minLen, maxLen; - minLen = maxLen = length[0]; - for(var i = 1; i < symCount; i++) { - if (length[i] > maxLen) maxLen = length[i]; - else if (length[i] < minLen) minLen = length[i]; - } - hufGroup = groups[j] = {}; - hufGroup.permute = new Int32Array(MAX_SYMBOLS); - hufGroup.limit = new Int32Array(MAX_HUFCODE_BITS + 1); - hufGroup.base = new Int32Array(MAX_HUFCODE_BITS + 1); - - hufGroup.minLen = minLen; - hufGroup.maxLen = maxLen; - var base = hufGroup.base; - var limit = hufGroup.limit; - var pp = 0; - for(var i: number = minLen; i <= maxLen; i++) - for(var t: any = 0; t < symCount; t++) - if (length[t] == i) hufGroup.permute[pp++] = t; - for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; - for(i = 0; i < symCount; i++) temp[length[i]]++; - pp = t = 0; - for(i = minLen; i < maxLen; i++) { - pp += temp[i]; - limit[i] = pp - 1; - pp <<= 1; - base[i+1] = pp - (t += temp[i]); - } - limit[maxLen] = pp + temp[maxLen] - 1; - base[minLen] = 0; + length[i] = t; + } + var minLen, maxLen; + minLen = maxLen = length[0]; + for (var i = 1; i < symCount; i++) { + if (length[i] > maxLen) maxLen = length[i]; + else if (length[i] < minLen) minLen = length[i]; + } + hufGroup = groups[j] = {}; + hufGroup.permute = new Int32Array(MAX_SYMBOLS); + hufGroup.limit = new Int32Array(MAX_HUFCODE_BITS + 1); + hufGroup.base = new Int32Array(MAX_HUFCODE_BITS + 1); + + hufGroup.minLen = minLen; + hufGroup.maxLen = maxLen; + var base = hufGroup.base; + var limit = hufGroup.limit; + var pp = 0; + for (var i: number = minLen; i <= maxLen; i++) + for (var t: any = 0; t < symCount; t++) + if (length[t] == i) hufGroup.permute[pp++] = t; + for (i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; + for (i = 0; i < symCount; i++) temp[length[i]]++; + pp = t = 0; + for (i = minLen; i < maxLen; i++) { + pp += temp[i]; + limit[i] = pp - 1; + pp <<= 1; + base[i + 1] = pp - (t += temp[i]); + } + limit[maxLen] = pp + temp[maxLen] - 1; + base[minLen] = 0; } - - for(var i = 0; i < 256; i++) { - this.mtfSymbol[i] = i; - this.byteCount[i] = 0; + + for (var i = 0; i < 256; i++) { + this.mtfSymbol[i] = i; + this.byteCount[i] = 0; } var runPos, count, symCount: number, selector; - runPos = count = symCount = selector = 0; - while(true) { - if (!(symCount--)) { - symCount = GROUP_SIZE - 1; - if (selector >= nSelectors) messageArg.Error("meow i'm a kitty, that's an error"); - hufGroup = groups[this.selectors[selector++]]; - base = hufGroup.base; - limit = hufGroup.limit; + runPos = count = symCount = selector = 0; + while (true) { + if (!symCount--) { + symCount = GROUP_SIZE - 1; + if (selector >= nSelectors) + messageArg.Error("meow i'm a kitty, that's an error"); + hufGroup = groups[this.selectors[selector++]]; + base = hufGroup.base; + limit = hufGroup.limit; + } + i = hufGroup.minLen; + j = bits(i); + while (true) { + if (i > hufGroup.maxLen) messageArg.Error("rawr i'm a dinosaur"); + if (j <= limit[i]) break; + i++; + j = (j << 1) | bits(1); + } + j -= base[i]; + if (j < 0 || j >= MAX_SYMBOLS) messageArg.Error("moo i'm a cow"); + var nextSym = hufGroup.permute[j]; + if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { + if (!runPos) { + runPos = 1; + t = 0; } - i = hufGroup.minLen; - j = bits(i); - while(true) { - if (i > hufGroup.maxLen) messageArg.Error("rawr i'm a dinosaur"); - if (j <= limit[i]) break; - i++; - j = (j << 1) | bits(1); - } - j -= base[i]; - if (j < 0 || j >= MAX_SYMBOLS) messageArg.Error("moo i'm a cow"); - var nextSym = hufGroup.permute[j]; - if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { - if (!runPos){ - runPos = 1; - t = 0; - } - if (nextSym == SYMBOL_RUNA) t += runPos; - else t += 2 * runPos; - runPos <<= 1; - continue; - } - if (runPos) { - runPos = 0; - if (count + t > bufsize) messageArg.Error("Boom."); - uc = this.symToByte[this.mtfSymbol[0]]; - this.byteCount[uc] += t; - while(t--) buf[count++] = uc; - } - if (nextSym > symTotal) break; - if (count >= bufsize) messageArg.Error("I can't think of anything. Error"); - i = nextSym - 1; - uc = this.mtfSymbol[i]; - for(var k: any = i-1; k>=0; k--) { - this.mtfSymbol[k+1] = this.mtfSymbol[k]; - } - this.mtfSymbol[0] = uc - uc = this.symToByte[uc]; - this.byteCount[uc]++; - buf[count++] = uc; + if (nextSym == SYMBOL_RUNA) t += runPos; + else t += 2 * runPos; + runPos <<= 1; + continue; + } + if (runPos) { + runPos = 0; + if (count + t > bufsize) messageArg.Error('Boom.'); + uc = this.symToByte[this.mtfSymbol[0]]; + this.byteCount[uc] += t; + while (t--) buf[count++] = uc; + } + if (nextSym > symTotal) break; + if (count >= bufsize) + messageArg.Error("I can't think of anything. Error"); + i = nextSym - 1; + uc = this.mtfSymbol[i]; + for (var k: any = i - 1; k >= 0; k--) { + this.mtfSymbol[k + 1] = this.mtfSymbol[k]; + } + this.mtfSymbol[0] = uc; + uc = this.symToByte[uc]; + this.byteCount[uc]++; + buf[count++] = uc; } - if (origPtr < 0 || origPtr >= count) messageArg.Error("I'm a monkey and I'm throwing something at someone, namely you"); + if (origPtr < 0 || origPtr >= count) + messageArg.Error( + "I'm a monkey and I'm throwing something at someone, namely you", + ); var j = 0; - for(var i = 0; i < 256; i++) { - k = j + this.byteCount[i]; - this.byteCount[i] = j; - j = k; + for (var i = 0; i < 256; i++) { + k = j + this.byteCount[i]; + this.byteCount[i] = j; + j = k; } - for(var i = 0; i < count; i++) { - uc = buf[i] & 0xff; - buf[this.byteCount[uc]] |= (i << 8); - this.byteCount[uc]++; + for (var i = 0; i < count; i++) { + uc = buf[i] & 0xff; + buf[this.byteCount[uc]] |= i << 8; + this.byteCount[uc]++; } - var pos = 0, current = 0, run = 0; + var pos = 0, + current = 0, + run = 0; if (count) { - pos = buf[origPtr]; - current = (pos & 0xff); - pos >>= 8; - run = -1; + pos = buf[origPtr]; + current = pos & 0xff; + pos >>= 8; + run = -1; } count = count; var copies, previous, outbyte; - while(count) { - count--; - previous = current; - pos = buf[pos]; - current = pos & 0xff; - pos >>= 8; - if (run++ == 3) { - copies = current; - outbyte = previous; - current = -1; - } else { - copies = 1; - outbyte = current; - } - while(copies--) { - crc = ((crc << 8) ^ this.crcTable[((crc>>24) ^ outbyte) & 0xFF])&0xFFFFFFFF; // crc32 - stream(outbyte); - } - if (current != previous) run = 0; + while (count) { + count--; + previous = current; + pos = buf[pos]; + current = pos & 0xff; + pos >>= 8; + if (run++ == 3) { + copies = current; + outbyte = previous; + current = -1; + } else { + copies = 1; + outbyte = current; + } + while (copies--) { + crc = + ((crc << 8) ^ this.crcTable[((crc >> 24) ^ outbyte) & 0xff]) & + 0xffffffff; // crc32 + stream(outbyte); + } + if (current != previous) run = 0; } - - crc = (crc ^ (-1)) >>> 0; - if ((crc|0) != (crcblock|0)) messageArg.Error("Error in bzip2: crc32 do not match"); - streamCRC = (crc ^ ((streamCRC << 1) | (streamCRC >>> 31))) & 0xFFFFFFFF; + + crc = (crc ^ -1) >>> 0; + if ((crc | 0) != (crcblock | 0)) + messageArg.Error('Error in bzip2: crc32 do not match'); + streamCRC = (crc ^ ((streamCRC << 1) | (streamCRC >>> 31))) & 0xffffffff; return streamCRC; }; -}; \ No newline at end of file +} diff --git a/ts/bzip2/index.ts b/ts/bzip2/index.ts index eb359f0..eb85a59 100644 --- a/ts/bzip2/index.ts +++ b/ts/bzip2/index.ts @@ -26,7 +26,13 @@ export function unbzip2Stream() { chunk.push(b); }; - streamCRC = bzip2Instance.decompress(bitReader, f, buf, bufsize, streamCRC); + streamCRC = bzip2Instance.decompress( + bitReader, + f, + buf, + bufsize, + streamCRC, + ); if (streamCRC === null) { // reset for next bzip2 header blockSize = 0; @@ -66,7 +72,10 @@ export function unbzip2Stream() { return bufferQueue.shift(); }); } - while (!broken && hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4)) { + while ( + !broken && + hasBytes - bitReader.bytesRead + 1 >= (25000 + 100000 * blockSize || 4) + ) { //console.error('decompressing with', hasBytes - bitReader.bytesRead + 1, 'bytes in buffer'); const result = await decompressAndPush(); if (!result) { @@ -86,7 +95,8 @@ export function unbzip2Stream() { await streamTools.push(result); } if (!broken) { - if (streamCRC !== null) this.emit('error', new Error('input stream ended prematurely')); + if (streamCRC !== null) + this.emit('error', new Error('input stream ended prematurely')); } }, }); diff --git a/ts/classes.archiveanalyzer.ts b/ts/classes.archiveanalyzer.ts index 8bff9df..45d5623 100644 --- a/ts/classes.archiveanalyzer.ts +++ b/ts/classes.archiveanalyzer.ts @@ -5,7 +5,10 @@ export interface IAnalyzedResult { fileType: plugins.fileType.FileTypeResult; isArchive: boolean; resultStream: plugins.smartstream.SmartDuplex; - decompressionStream: plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract; + decompressionStream: + | plugins.stream.Transform + | plugins.stream.Duplex + | plugins.tarStream.Extract; } export class ArchiveAnalyzer { @@ -25,14 +28,15 @@ export class ArchiveAnalyzer { 'application/x-bzip2', // Add other archive mime types here ]); - + return archiveMimeTypes.has(mimeType); } - private async getDecompressionStream( - mimeTypeArg: plugins.fileType.FileTypeResult['mime'] - ): Promise { + mimeTypeArg: plugins.fileType.FileTypeResult['mime'], + ): Promise< + plugins.stream.Transform | plugins.stream.Duplex | plugins.tarStream.Extract + > { switch (mimeTypeArg) { case 'application/gzip': return this.smartArchiveRef.gzipTools.getDecompressionStream(); @@ -51,13 +55,18 @@ export class ArchiveAnalyzer { public getAnalyzedStream() { let firstRun = true; const resultStream = plugins.smartstream.createPassThrough(); - const analyzerstream = new plugins.smartstream.SmartDuplex({ + const analyzerstream = new plugins.smartstream.SmartDuplex< + Buffer, + IAnalyzedResult + >({ readableObjectMode: true, writeFunction: async (chunkArg: Buffer, streamtools) => { if (firstRun) { firstRun = false; const fileType = await plugins.fileType.fileTypeFromBuffer(chunkArg); - const decompressionStream = await this.getDecompressionStream(fileType?.mime as any); + const decompressionStream = await this.getDecompressionStream( + fileType?.mime as any, + ); /** * analyzed stream emits once with this object */ @@ -75,7 +84,7 @@ export class ArchiveAnalyzer { finalFunction: async (tools) => { resultStream.push(null); return null; - } + }, }); return analyzerstream; } diff --git a/ts/classes.bzip2tools.ts b/ts/classes.bzip2tools.ts index d8e6415..81a4bbc 100644 --- a/ts/classes.bzip2tools.ts +++ b/ts/classes.bzip2tools.ts @@ -13,4 +13,4 @@ export class Bzip2Tools { getDecompressionStream() { return unbzip2Stream(); } -} \ No newline at end of file +} diff --git a/ts/classes.gziptools.ts b/ts/classes.gziptools.ts index 448d4bd..528d901 100644 --- a/ts/classes.gziptools.ts +++ b/ts/classes.gziptools.ts @@ -1,5 +1,5 @@ import type { SmartArchive } from './classes.smartarchive.js'; -import * as plugins from './plugins.js' +import * as plugins from './plugins.js'; // This class wraps fflate's gunzip in a Node.js Transform stream export class CompressGunzipTransform extends plugins.stream.Transform { @@ -7,7 +7,11 @@ export class CompressGunzipTransform extends plugins.stream.Transform { super(); } - _transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) { + _transform( + chunk: Buffer, + encoding: BufferEncoding, + callback: plugins.stream.TransformCallback, + ) { plugins.fflate.gunzip(chunk, (err, decompressed) => { if (err) { callback(err); @@ -26,7 +30,11 @@ export class DecompressGunzipTransform extends plugins.stream.Transform { super(); } - _transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) { + _transform( + chunk: Buffer, + encoding: BufferEncoding, + callback: plugins.stream.TransformCallback, + ) { // Use fflate's gunzip function to decompress the chunk plugins.fflate.gunzip(chunk, (err, decompressed) => { if (err) { @@ -41,10 +49,8 @@ export class DecompressGunzipTransform extends plugins.stream.Transform { } } - export class GzipTools { - constructor() { - } + constructor() {} public getCompressionStream() { return new CompressGunzipTransform(); diff --git a/ts/classes.smartarchive.ts b/ts/classes.smartarchive.ts index 4f3905e..cff52cd 100644 --- a/ts/classes.smartarchive.ts +++ b/ts/classes.smartarchive.ts @@ -6,7 +6,10 @@ import { GzipTools } from './classes.gziptools.js'; import { TarTools } from './classes.tartools.js'; import { ZipTools } from './classes.ziptools.js'; -import { ArchiveAnalyzer, type IAnalyzedResult } from './classes.archiveanalyzer.js'; +import { + ArchiveAnalyzer, + type IAnalyzedResult, +} from './classes.archiveanalyzer.js'; import type { from } from '@push.rocks/smartrx/dist_ts/smartrx.plugins.rxjs.js'; @@ -18,14 +21,19 @@ export class SmartArchive { return smartArchiveInstance; } - public static async fromArchiveFile(filePathArg: string): Promise { + public static async fromArchiveFile( + filePathArg: string, + ): Promise { const smartArchiveInstance = new SmartArchive(); smartArchiveInstance.sourceFilePath = filePathArg; return smartArchiveInstance; } public static async fromArchiveStream( - streamArg: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform + streamArg: + | plugins.stream.Readable + | plugins.stream.Duplex + | plugins.stream.Transform, ): Promise { const smartArchiveInstance = new SmartArchive(); smartArchiveInstance.sourceStream = streamArg; @@ -41,13 +49,19 @@ export class SmartArchive { public sourceUrl: string; public sourceFilePath: string; - public sourceStream: plugins.stream.Readable | plugins.stream.Duplex | plugins.stream.Transform; + public sourceStream: + | plugins.stream.Readable + | plugins.stream.Duplex + | plugins.stream.Transform; public archiveName: string; public singleFileMode: boolean = false; public addedDirectories: string[] = []; - public addedFiles: (plugins.smartfile.SmartFile | plugins.smartfile.StreamFile)[] = []; + public addedFiles: ( + | plugins.smartfile.SmartFile + | plugins.smartfile.StreamFile + )[] = []; public addedUrls: string[] = []; constructor() {} @@ -81,24 +95,35 @@ export class SmartArchive { // return archiveStream; } - public async exportToFs(targetDir: string, fileNameArg?: string): Promise { + public async exportToFs( + targetDir: string, + fileNameArg?: string, + ): Promise { const done = plugins.smartpromise.defer(); const streamFileStream = await this.exportToStreamOfStreamFiles(); streamFileStream.pipe( new plugins.smartstream.SmartDuplex({ objectMode: true, - writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => { + writeFunction: async ( + streamFileArg: plugins.smartfile.StreamFile, + streamtools, + ) => { const done = plugins.smartpromise.defer(); - console.log(streamFileArg.relativeFilePath ? streamFileArg.relativeFilePath : 'no relative path'); + console.log( + streamFileArg.relativeFilePath + ? streamFileArg.relativeFilePath + : 'no relative path', + ); const streamFile = streamFileArg; const readStream = await streamFile.createReadStream(); await plugins.smartfile.fs.ensureDir(targetDir); const writePath = plugins.path.join( targetDir, - streamFile.relativeFilePath || fileNameArg + streamFile.relativeFilePath || fileNameArg, ); await plugins.smartfile.fs.ensureDir(plugins.path.dirname(writePath)); - const writeStream = plugins.smartfile.fsStream.createWriteStream(writePath); + const writeStream = + plugins.smartfile.fsStream.createWriteStream(writePath); readStream.pipe(writeStream); writeStream.on('finish', () => { done.resolve(); @@ -108,15 +133,16 @@ export class SmartArchive { finalFunction: async () => { done.resolve(); }, - }) + }), ); return done.promise; } public async exportToStreamOfStreamFiles() { - const streamFileIntake = new plugins.smartstream.StreamIntake({ - objectMode: true, - }); + const streamFileIntake = + new plugins.smartstream.StreamIntake({ + objectMode: true, + }); const archiveStream = await this.getArchiveStream(); const createAnalyzedStream = () => this.archiveAnalyzer.getAnalyzedStream(); @@ -125,15 +151,21 @@ export class SmartArchive { plugins.smartstream.createTransformFunction( async (analyzedResultChunk) => { if (analyzedResultChunk.fileType?.mime === 'application/x-tar') { - const tarStream = analyzedResultChunk.decompressionStream as plugins.tarStream.Extract; + const tarStream = + analyzedResultChunk.decompressionStream as plugins.tarStream.Extract; tarStream.on('entry', async (header, stream, next) => { if (header.type === 'directory') { - console.log(`tar stream directory: ${header.name} ... skipping!`); + console.log( + `tar stream directory: ${header.name} ... skipping!`, + ); next(); return; } console.log(`tar stream file: ${header.name}`); - const streamfile = plugins.smartfile.StreamFile.fromStream(stream, header.name); + const streamfile = plugins.smartfile.StreamFile.fromStream( + stream, + header.name, + ); streamFileIntake.push(streamfile); stream.on('end', function () { next(); // ready for next entry @@ -143,20 +175,30 @@ export class SmartArchive { console.log('finished'); streamFileIntake.signalEnd(); }); - analyzedResultChunk.resultStream.pipe(analyzedResultChunk.decompressionStream); + analyzedResultChunk.resultStream.pipe( + analyzedResultChunk.decompressionStream, + ); } else if (analyzedResultChunk.fileType?.mime === 'application/zip') { analyzedResultChunk.resultStream .pipe(analyzedResultChunk.decompressionStream) - .pipe(new plugins.smartstream.SmartDuplex({ - objectMode: true, - writeFunction: async (streamFileArg: plugins.smartfile.StreamFile, streamtools) => { - streamFileIntake.push(streamFileArg); - }, - finalFunction: async () => { - streamFileIntake.signalEnd(); - } - })); - } else if (analyzedResultChunk.isArchive && analyzedResultChunk.decompressionStream) { + .pipe( + new plugins.smartstream.SmartDuplex({ + objectMode: true, + writeFunction: async ( + streamFileArg: plugins.smartfile.StreamFile, + streamtools, + ) => { + streamFileIntake.push(streamFileArg); + }, + finalFunction: async () => { + streamFileIntake.signalEnd(); + }, + }), + ); + } else if ( + analyzedResultChunk.isArchive && + analyzedResultChunk.decompressionStream + ) { analyzedResultChunk.resultStream .pipe(analyzedResultChunk.decompressionStream) .pipe(createAnalyzedStream()) @@ -164,7 +206,7 @@ export class SmartArchive { } else { const streamFile = plugins.smartfile.StreamFile.fromStream( analyzedResultChunk.resultStream, - analyzedResultChunk.fileType?.ext + analyzedResultChunk.fileType?.ext, ); streamFileIntake.push(streamFile); streamFileIntake.signalEnd(); @@ -172,7 +214,7 @@ export class SmartArchive { }, { objectMode: true, - } + }, ); archiveStream.pipe(createAnalyzedStream()).pipe(createUnpackStream()); diff --git a/ts/classes.tartools.ts b/ts/classes.tartools.ts index 5e7a6f8..03cb915 100644 --- a/ts/classes.tartools.ts +++ b/ts/classes.tartools.ts @@ -18,7 +18,7 @@ export class TarTools { | plugins.smartfile.StreamFile; byteLength?: number; filePath?: string; - } + }, ): Promise { return new Promise(async (resolve, reject) => { let fileName: string | null = null; @@ -28,7 +28,8 @@ export class TarTools { } else if (optionsArg.content instanceof plugins.smartfile.SmartFile) { fileName = (optionsArg.content as plugins.smartfile.SmartFile).relative; } else if (optionsArg.content instanceof plugins.smartfile.StreamFile) { - fileName = (optionsArg.content as plugins.smartfile.StreamFile).relativeFilePath; + fileName = (optionsArg.content as plugins.smartfile.StreamFile) + .relativeFilePath; } else if (optionsArg.filePath) { fileName = optionsArg.filePath; } @@ -47,9 +48,11 @@ export class TarTools { contentByteLength = await optionsArg.content.getSize(); // assuming SmartFile has getSize method } else if (optionsArg.content instanceof plugins.smartfile.StreamFile) { contentByteLength = await optionsArg.content.getSize(); // assuming StreamFile has getSize method - } else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) { + } else if ( + optionsArg.content instanceof plugins.smartstream.stream.Readable + ) { console.warn( - '@push.rocks/smartarchive: When streaming, it is recommended to provide byteLength, if known.' + '@push.rocks/smartarchive: When streaming, it is recommended to provide byteLength, if known.', ); } else if (optionsArg.filePath) { const fileStat = await plugins.smartfile.fs.stat(optionsArg.filePath); @@ -63,12 +66,18 @@ export class TarTools { if (Buffer.isBuffer(optionsArg.content)) { content = plugins.smartstream.stream.Readable.from(optionsArg.content); } else if (typeof optionsArg.content === 'string') { - content = plugins.smartstream.stream.Readable.from(Buffer.from(optionsArg.content)); + content = plugins.smartstream.stream.Readable.from( + Buffer.from(optionsArg.content), + ); } else if (optionsArg.content instanceof plugins.smartfile.SmartFile) { - content = plugins.smartstream.stream.Readable.from(optionsArg.content.contents); + content = plugins.smartstream.stream.Readable.from( + optionsArg.content.contents, + ); } else if (optionsArg.content instanceof plugins.smartfile.StreamFile) { content = await optionsArg.content.createReadStream(); - } else if (optionsArg.content instanceof plugins.smartstream.stream.Readable) { + } else if ( + optionsArg.content instanceof plugins.smartstream.stream.Readable + ) { content = optionsArg.content; } @@ -87,7 +96,7 @@ export class TarTools { } else { resolve(); } - } + }, ); content.pipe(entry); @@ -100,7 +109,10 @@ export class TarTools { * @param directoryPath */ public async packDirectory(directoryPath: string) { - const fileTree = await plugins.smartfile.fs.listFileTree(directoryPath, '**/*'); + const fileTree = await plugins.smartfile.fs.listFileTree( + directoryPath, + '**/*', + ); const pack = await this.getPackStream(); for (const filePath of fileTree) { const absolutePath = plugins.path.join(directoryPath, filePath); diff --git a/ts/classes.ziptools.ts b/ts/classes.ziptools.ts index be9739e..a455f4e 100644 --- a/ts/classes.ziptools.ts +++ b/ts/classes.ziptools.ts @@ -1,33 +1,39 @@ import type { SmartArchive } from './classes.smartarchive.js'; import * as plugins from './plugins.js'; -class DecompressZipTransform extends plugins.smartstream.SmartDuplex { +class DecompressZipTransform extends plugins.smartstream + .SmartDuplex { private streamtools: plugins.smartstream.IStreamTools; private unzipper = new plugins.fflate.Unzip(async (fileArg) => { let resultBuffer: Buffer; fileArg.ondata = async (flateError, dat, final) => { - resultBuffer? resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)]) - : resultBuffer = Buffer.from(dat); + resultBuffer + ? (resultBuffer = Buffer.concat([resultBuffer, Buffer.from(dat)])) + : (resultBuffer = Buffer.from(dat)); if (final) { - const streamFile = plugins.smartfile.StreamFile.fromBuffer(resultBuffer); + const streamFile = + plugins.smartfile.StreamFile.fromBuffer(resultBuffer); streamFile.relativeFilePath = fileArg.name; this.streamtools.push(streamFile); } - } + }; fileArg.start(); }); constructor() { super({ objectMode: true, writeFunction: async (chunkArg, streamtoolsArg) => { - this.streamtools? null : this.streamtools = streamtoolsArg; - this.unzipper.push(Buffer.isBuffer(chunkArg) ? chunkArg : Buffer.from(chunkArg), false); + this.streamtools ? null : (this.streamtools = streamtoolsArg); + this.unzipper.push( + Buffer.isBuffer(chunkArg) ? chunkArg : Buffer.from(chunkArg), + false, + ); }, finalFunction: async () => { this.unzipper.push(Buffer.from(''), true); await plugins.smartdelay.delayFor(0); await this.streamtools.push(null); - } + }, }); this.unzipper.register(plugins.fflate.UnzipInflate); } @@ -42,7 +48,11 @@ export class CompressZipTransform extends plugins.stream.Transform { this.files = {}; } - _transform(chunk: Buffer, encoding: BufferEncoding, callback: plugins.stream.TransformCallback) { + _transform( + chunk: Buffer, + encoding: BufferEncoding, + callback: plugins.stream.TransformCallback, + ) { // Simple example: storing chunks in memory before finalizing ZIP in _flush this.files['file.txt'] = new Uint8Array(chunk); callback(); @@ -61,8 +71,7 @@ export class CompressZipTransform extends plugins.stream.Transform { } export class ZipTools { - constructor() { - } + constructor() {} public getCompressionStream() { return new CompressZipTransform(); diff --git a/ts/paths.ts b/ts/paths.ts index 857a644..cc2c0fd 100644 --- a/ts/paths.ts +++ b/ts/paths.ts @@ -2,6 +2,6 @@ import * as plugins from './plugins.js'; export const packageDir = plugins.path.join( plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url), - '../' + '../', ); export const nogitDir = plugins.path.join(packageDir, './.nogit'); diff --git a/ts/plugins.ts b/ts/plugins.ts index 762a618..59ab1dc 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -15,7 +15,17 @@ import * as smartstream from '@push.rocks/smartstream'; import * as smartrx from '@push.rocks/smartrx'; import * as smarturl from '@push.rocks/smarturl'; -export { smartfile, smartdelay, smartpath, smartpromise, smartrequest, smartunique, smartstream, smartrx, smarturl }; +export { + smartfile, + smartdelay, + smartpath, + smartpromise, + smartrequest, + smartunique, + smartstream, + smartrx, + smarturl, +}; // third party scope import * as fileType from 'file-type'; diff --git a/tsconfig.json b/tsconfig.json index dfe5a55..575a85e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,9 +6,9 @@ "module": "NodeNext", "moduleResolution": "NodeNext", "esModuleInterop": true, - "verbatimModuleSyntax": true + "verbatimModuleSyntax": true, + "baseUrl": ".", + "paths": {} }, - "exclude": [ - "dist_*/**/*.d.ts" - ] + "exclude": ["dist_*/**/*.d.ts"] }