From a32ed0facd7cc240d3685a52441200f2b3d07abf Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 18 Aug 2025 01:01:02 +0000 Subject: [PATCH] feat(classes.smartarchive): Support URL web streams, add recursive archive unpacking and filesystem export, and improve ZIP decompression robustness --- .../document_symbols_cache_v23-06-25.pkl | Bin 0 -> 56731 bytes .serena/project.yml | 68 ++++++++++++++++++ changelog.md | 8 +++ ts/00_commitinfo_data.ts | 2 +- ts/classes.smartarchive.ts | 7 +- ts/classes.ziptools.ts | 4 +- 6 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 .serena/cache/typescript/document_symbols_cache_v23-06-25.pkl create mode 100644 .serena/project.yml diff --git a/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl b/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6745bd5215166f70d70e4a565ddaeb15d3f4a835 GIT binary patch literal 56731 zcmeHQYm6MnaXyKp#5*1zq9~c7EORJQguZM?&_ah`0}U!ejxrg+Zt&)^=PbEaihqK z3emJ1*4?n^moIpQdbIVhTZz0@)7iCS$9>y({77m0E_brH-Q7`qpt!SCn%Xvb-v@W@ z+%>hWHPITuXDzI{(@=l-6h3F4cuuQ%Q^~8lLnn*TPnd zB6w##{_ny{x$3o!4L1jiz8kv5x)-*N4K$hyJZJ@V{bk3#>@PodL!;4L7I_t~STEPA zk0np?heefPsIs)-uTZ%=puuAJtV${LslZ7u-s}sGF7g4yT``H zrmOX_l3RDjDzzx8cu_Pq(}?^+SSy~7#&nX5<&&zF0P3%44!e_4tphe9UZmHyz4wge8R4pufQ3CSK1myb&$e0^7K*qAclnj^y ziuwSuzs_Ii-;HnEFwcPc8~v^Rrujn8AN4mUs0_uZ90uy=;IWP1fw_+wcwm{r19P|b z0FPZfcnrpPoDM5FaNJMez}(#i99X8{z#NWJZuU6D1IKbHfcI71%H=B{0Ht7}V`dWp z1#=G=pkSE-1#_!=0Lr60pe&J~9LEX93dmLh2rc{<5p zPEInY+rVtLuv7!(L^9tp^6(Q-c_?j)x$VZovo)5b%onlEU_Z_S_R^^`2&co1YOH@H zkWjp*0EERUBtGN@4M=DW%-z-lNPdYB{Ik|0P^P2hK^rK@A+t9WqeK)>zX6E_sy)m8BQMa@}h`aK;Iv zc#ty(4y}PXIdD1)s?$(;MLH@mx68mgTSLu~Ut$wfzsZC55b$8B9sivncwp{B1|HcO zivRfV_&O6FYE;_CFYtJa2amycJjQmibzXmhz%c>2B?cU{2IiDy z%Eunx@~PydS||2^1uPZr<=UN(Oo`&QDeTlE9{Ot6VpZv z%o9`MFL5AI-Mb15r@WSLK+d#vu!gdAFo$v;*iWhuOQpTtP8NjFO6hKx+iA#xY>oFN zvLIM+!@vNSIuR_xhqE~ERuw^|sC&{m(^j{L2tJrQVc?Uk@qympvxW_yl>(pB)fo_u z?YQRxLFbE*yV*d8*1+8R1v-*PJDOatlUHC4RTbUj%GOAg-CzSJgO0Hg;Da-j#<_A; zF6KnB?Nty&8HY8OXsQ-YCkDt!r+MrWV5W@t*C1!wBK8`ZAnC$Y@G?AC86}^E+${#3 zXbsGvb1$APgI%0-I><(+j@`WI$l&h#)iruEReR7rB?_jn42^}$<|1hw1#Iq4+2?u4ne1b)p#<6;Qlz3a0ISj+-}oH7V{Q7zYmi%R(8gXv2)7X7V_M$~ zX4z;nBtV+V9mdZB4y#gBFgIbKlC7b1BUV&C!GH>wNxZ|X&d_OLeXOO!tV;v<1)!)h zfIkU2bDY_0^k7Q=c{ZrJ@$S9^P$*re1vyjKp*1kqBVFh748S1cvntXLGIg5OJl)99 zX)#*0WtXGXx_drHi>d)vDbxLF$lYX!Us?lmQnwWE>8QQV#LrCjU}5fd;|5R57NGup_{C$`SSm zVs;A95} z&xZ*xnA>9jldYlLgRH=OhXEMg!SpsmkHuhGJ{8uc06_8I2q}7l!2NKq} zJ=N$jcaIoI;x(({jjZxLM`72kTP@~HPPai@HKH(QZl=lBP`1H4oFnRg7{~xq2cs1J zcZMDdir%6z4i%7;b=Cj@73(;VGg${~DAvK;rt4&#yP&ON9n6`mldYjxXUUC>TF349 z@OfU6;ANeq3_TXCvrJIr*l*Km}gDxrd?0f^Sxw^O9pF?Ig}Fpr7KLLy$8$2Wu#;4|8h;=Ojx@=((b= zpjzL4XfmA6c9=8Q_Odn5021cSgPaU=m<|5HMp$VF%3cBr<{p7h*4NvzHPS7g0V^nv zF@VCW@$F&gu^2^zXwtOVIerfiP7NQ-p(dx35zExham=Zms(c2e%ZA9Zs9Zf)@lgHE z-@scSuwZV~fCbAGESO8}8Ac6nnh};-BP_?uKWRYLw1El@5vw;ADITH{pQ-ENu7l8Z< z10WukbK74s&o-vN?s&sr?ceL~^bhza{ImW!*zfs_|AOE0f64!n{}umh{x|%;@&C#H ze(e7ahQZ$uti}YFeS)o+;0~Q&KPJa9c>u&BGV5&WU0 z7;I@`kY^MV4|BlLs}H`~c@pPdY{ z81z;<8RYMS2K8hyuu0GxGJJcU$q;-K9@q%}4$|hPnZ{(LTy)IP1iPK69)cNgcEZ`? zoSk@3qk(>4zZ>+o+0mE9AI(Ew{%)c#i*M~2`rl(j|8X6Cw6~(&&IsADik_E396$tr zcAjx8zothEM=jLS!U=Rkto!Z6lEttXXdf!_cavDM7zJH#hRT0263fwv1)4q3DtC-z zq84<6{U$s1viN~K?B(w!_Ocj`)%FPcn-|RQY@v&(?FsfGicS&?0%TLwyXeSuw4`*T z?w6xcU1X7X3EhxltDO|G7`0&GIWFZ zMmyrN7=ks}5tqN4h|A*M%@XSv5g(O^s~8uO@As!cFS0B@lq0s53MFsK3m>{rE0^|;5HV;3svtDTVT&n2gT)37d{GR? zU+gryxzSXj9-;C&KtYhO*M!MXhpxo_1I6F_rS=$jRJ=O&697%N2 zE{S9@#OSc2-M$&@Annm8_&6il4}lI$BAb9zaxc2&x`SF+A(@2)*yOOTA)9C{5JHt4 z*)(n+16~K&l+#QWE7^2CQ`Twm?iGf(l*PO97^QtPqBoY2mRwU-osm(lZ_3g`mJpuE z8L4ON1dzoL&BQ)Z+czU4m6ZTL#Yli{D*8)0M%CbzhhiT~BgD|*DUUFr6N0LfIdpj3 zP6t^Gr|a!>Xy2@Nboe<&I^3y12Rf+CzVMuGKxx5Px?%h$?0`>*Vu-h42Rx^lELQqE zxSClR|8tChUrxi?Xd3ejM1w5^wQ_YNRmf0dgH=k{RRu9xw~%5*CD7zV+*ce5xDUX2 z<{1aQbZ7_z3AJWK_a?z)x*_P3c7n>{(|H7K-walnmCnIR(AOCWdRU7?qmeY;Q;no3 zOhu>klm3rmm@!c@lJWwfP8X4F2+fA=7ON|0w%(Zjiya1G=Vqz*l$B;*VWio%tYPFh zd8U!$%p!U4xE!k)a-Q#l&&yMpqAb%DIVC_nC9G%%33+GEdBVoD#56!#{c_bs{#@^Df*y4yJ^e1gIoDBV)m zB{s}4oH9|@#(qq=$nX{ju8}mwwVZ^fW1bYX#ro&L#!SV#G@?9f88=VL8GW%EJ0DGw z^~J7TOHcX6ky8!f!R*|T{M;A21&i2C(R|nLarva#(fB`dItjgcBdiL|QG2_-NvLgv zK%&OyI2NlyUpKdHy&;No*TQ5ob1h63qt8RA@yfAi#1$lc%W5r$w)6-lsU<3<6gB<}f;p(H7pY|#|EA<_jqk-jL3A<&|oNIA`9 zF@#aejAuO#2%$lcUo8R+BYN11>fZy%>Tj2J#AO{aq_(bJ%yp2zXeXL1hT~y& zqP1^EmO@yGb_XNT+79x$E;+p-a9(LRUtbl)5OB?ob51i^tfV~n5?FEG$cXdj6}Ma6 zKBrQvynwL?M3yd$fNdma=EuCKQK=vB8`bl;tp)xs>eCeydwn#f#xKu!55vZQIvHie z9j`_BBqBRUC%$5*yex*ZhIY!gZ$_3%SSi1qk@Am%qf?Lv^1#S?i{YTt~QW2{vAF-9u=XcP;1l)!VC z14N@Ojt>hmURb~thGb=qRVIFj4~aAP!yys5SyOygvdXq4W=pHeSe|l5BnEYuznndL z_Bfkh$tAwZ6$342`5xV`$r9UBAV=jgnv=HL?QFQJ;X3OS3I!)RU!IwPgIDXGvZ$eQ zC>cY>fpLXSF;i`wHR|Pc2wB4HOZM%$H+epo$mpD4w8PL9sgL8WzjA33Uzb5cG?2lr zOPgQ?WPYU(>0i7Yol2#SL4b*<%9;3H5TeT5EAUBCoU5*OZkA3ctON8M-vDjSv?Ha? z#qt6o58{rS%%9<>T>FmF)`(h6D4AoGza&UzTIDTKoNJZmG?T@>v&!p?lzfGGDK615 zGSt?j2g(%KXp!L%&^2IZIIa=1I>{FB!F%Sn7N;|rMYqpPWH^ag>J?DF3af0oJT2!o zcV-681;@9vk6qdd(^qDW6B9yeAk3G2LF8`wbnQ}uEy+oX{^adW$Ls$6$vcq)@m~<= zgo(^NWh(F+;vIJF^#_hghxE}lEOqCK*e7ADWijB%_s=?BCGwo>9s6*^Ff;OP-OxKr zzRYpv)^}^;x$O)|xFLUsNK)fHK0O&&I5675g_GG(m=W~J=VhkFTqOuwnD)Z#=v)}7 z9LZzY&c==1dYVkLsrQU!TcaEq6*~?R6cY~4-sJ{0bx|XS+Eqd}-EKlvWDdqRd5+zo zst>z&>wqP%%dn22QIQXKv@N3Tms1pvCCV4;c7yNntx%qMa&TZE%Mw7|c!67jnd}wP zx3h^xnd|2sI=YYeXF<=YUf!#`@X}>r86%nTf4#!Kujuu?G}P5cWP)XwYf|Ao;iVz{ z?JIiPQe7t_udnE(i(YC!{Hb(dH1QGJPhuphLZ?|X&ksS^79ZAE^b$z%?>tx#eMK(; zw3FthiFDLgJ5|pyDRf0Ih*-4isEbbv;nl*JGl-`=nG&EUc?wUO(2bUSjbCcnhz(`?%I1mS&R ztaidD7(eKuIVpgNkn7Kg=NrM`4-wk|r|*KAj`pSpVeI25UesVe|MVw}=GF9)obyit z1S6CPIok9C0z4LT!rY(RPIRrIZJJ6nbawitQQE47-*#bXeFOKJ!^gh zE@J6NT&!0L6MH*nE7TET*P1719}&g5Cue0d^W>~7#;D`wZf^Y|V$!i3N5E0h1Je;8 zi}&Xl-0ho@lXkz%X>k84qkXzd-RTUacs5&t<*zNr<@O|?cydd2g?_^wEYo4Aj zNcWy81iC~?GhVNo@8f5Hb#1rx3GIYLR&SEt)c7n!9LRtOcT)~$ZlfmSDn|ydvdiGr z9b~XrfxX)|Bd0X}lv5c072hamZjRTU z_K9l_l8eECwb5aCYT6Bx3t!{$j&4!b_(p%m_@)0K2Rqz-UdIb);uuHTO~|T4!*eCdV!uTWfP9 zTdoHuBPC}nud$D1r^8s5&CIbZi{U!)`5Vi3*~hXh&K=9`n~|~n9nP`*eMSy{Z;j=D z;rr4vo7LQ%H=L!7B8FSM%0Yu4#zh4TKkQD@i;l!EBK((r0g@_a=T2B#kn$}eKR5x1|T>d+v?^L zd^l5>iuaaGxRLQo|GdNe4mW6xG{VXcZf1uLz}OXCnDfA0VFr}yDt^l@`_MhqVd%BpI&jhjjqZ;l$%X0E^Smqqr$EI+ zPWVWzWZX`YgMPt|zAVnY5Lo_hqA!by>zNh(lZ@ym7XnjMcRQ01f!>21L*NE+;XI)p z251?b?YiRs&2|dN;@qHU@^_O0vbc8?m|\NK1()(DF+U!(siiRrBaKP-T^X%jh& zf3F>OS)3awOa5-cE{lma!OHjnBkWHR*afEA+NXE;`MoG%im43WidtpNo^t4G4dQE z+Uq3R37&Fhh84PaNs>;?i<9}fE7UjGQJ2NJsLS6?)Mat+P=B!ps3&+z)SoENDAYYR zZ`kkgU8#QNoz(*a;#v!*SaG9>EOEuw1uT@F8s9g5^Z;n%$xzQ(FoMDV!{$)J{XHGN z3ia9_@@23tzEaxfvB?$1Q|f@>i(GN9^?=x&Sqn*j zP^sjV;M%D;T433WlJ7Y?0a%@#K#zhH11{FYF%NY{vd+eM#T$Rt7s7C@dkV_5<3Xc2 zPX@5FFNe3s4xF%2tk**H8O_E_xOp@D;%s)ns{{u^e9x(c<#QN-BsmEp7rx?JW7;>w zOZk0Yl-wVLq!D}y#LkkmEB`S<4F9B`PCZfcMd|ED?u!EL?8p@?a2CH|A3Ug6r_UHghD%n|ul-Z^{eBWo2MYNUq18Vbdg-s!$M63WAc5 zy^gSdUtu*W#CKD1UvI0?$FlqqPe~x%)TN7w@QXXwzpt5u^;j-PfefH z-NjkC;!;c*{d2Z%eU18zjL(!zrVk=^v9{hnXIpw~4rlH9ai;olruuQF5`AXA zqFASM2?`~b#FES2)*=oj*lP6yie|2wML_XizUC}`XV;*U*C-V@8!cZd#zcJnG$-OboUAEPCrU(>V5C-obAwT z%U^fA5xfNpE5Wxg`7S2^iOC`es1vNjWGyCJFnIuz{g|A<oK_xlYN*R$K)&~J|+!Jp2g%-ka!cV O_&k$)t#z%{DEu#|jzl#8 literal 0 HcmV?d00001 diff --git a/.serena/project.yml b/.serena/project.yml new file mode 100644 index 0000000..50e7e31 --- /dev/null +++ b/.serena/project.yml @@ -0,0 +1,68 @@ +# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby) +# * For C, use cpp +# * For JavaScript, use typescript +# Special requirements: +# * csharp: Requires the presence of a .sln file in the project folder. +language: typescript + +# whether to use the project's gitignore file to ignore files +# Added on 2025-04-07 +ignore_all_files_in_gitignore: true +# list of additional paths to ignore +# same syntax as gitignore, so you can use * and ** +# Was previously called `ignored_dirs`, please update your config if you are using that. +# Added (renamed) on 2025-04-07 +ignored_paths: [] + +# whether the project is in read-only mode +# If set to true, all editing tools will be disabled and attempts to use them will result in an error +# Added on 2025-04-18 +read_only: false + + +# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. +# Below is the complete list of tools for convenience. +# To make sure you have the latest list of tools, and to view their descriptions, +# execute `uv run scripts/print_tool_overview.py`. +# +# * `activate_project`: Activates a project by name. +# * `check_onboarding_performed`: Checks whether project onboarding was already performed. +# * `create_text_file`: Creates/overwrites a file in the project directory. +# * `delete_lines`: Deletes a range of lines within a file. +# * `delete_memory`: Deletes a memory from Serena's project-specific memory store. +# * `execute_shell_command`: Executes a shell command. +# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced. +# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). +# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). +# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. +# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. +# * `initial_instructions`: Gets the initial instructions for the current project. +# Should only be used in settings where the system prompt cannot be set, +# e.g. in clients you have no control over, like Claude Desktop. +# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +# * `insert_at_line`: Inserts content at a given line in a file. +# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +# * `list_dir`: Lists files and directories in the given directory (optionally with recursion). +# * `list_memories`: Lists memories in Serena's project-specific memory store. +# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). +# * `read_file`: Reads a file within the project directory. +# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +# * `remove_project`: Removes a project from the Serena configuration. +# * `replace_lines`: Replaces a range of lines within a file with new content. +# * `replace_symbol_body`: Replaces the full definition of a symbol. +# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. +# * `search_for_pattern`: Performs a search for a pattern in the project. +# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +# * `switch_modes`: Activates modes by providing a list of their names +# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. +# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. +excluded_tools: [] + +# initial prompt for the project. It will always be given to the LLM upon activating the project +# (contrary to the memories, which are loaded on demand). +initial_prompt: "" + +project_name: "smartarchive" diff --git a/changelog.md b/changelog.md index 366f94f..9dd8e1e 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,13 @@ # Changelog +## 2025-08-18 - 4.1.0 - feat(classes.smartarchive) +Support URL web streams, add recursive archive unpacking and filesystem export, and improve ZIP decompression robustness + +- ts/classes.smartarchive.ts: add exportToFs(targetDir, fileName?) to write extracted StreamFile objects to the filesystem (ensures directories, logs relative paths, waits for write completion). +- ts/classes.smartarchive.ts: implement exportToStreamOfStreamFiles with recursive unpacking pipeline that handles application/x-tar (tar-stream Extract), application/zip (fflate unzip), nested archives and StreamIntake for StreamFile results. +- ts/classes.smartarchive.ts: improve getArchiveStream() for URL sources by using SmartRequest.create().url(...).get() and converting the returned Web stream into a Node Readable stream. +- ts/classes.ziptools.ts: make ZIP decompression writeFunction more robust — accept non-Buffer chunks, coerce to Buffer before pushing to fflate.Unzip, and loosen the writeFunction typing to handle incoming ArrayBuffer-like data. + ## 2024-10-13 - 4.0.39 - fix(core) Fix dependencies and update documentation. diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 379f94e..f5a4608 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartarchive', - version: '4.0.39', + version: '4.1.0', description: 'A library for working with archive files, providing utilities for compressing and decompressing data.' } diff --git a/ts/classes.smartarchive.ts b/ts/classes.smartarchive.ts index 94793d5..4f3905e 100644 --- a/ts/classes.smartarchive.ts +++ b/ts/classes.smartarchive.ts @@ -60,7 +60,12 @@ export class SmartArchive { return this.sourceStream; } if (this.sourceUrl) { - const urlStream = await plugins.smartrequest.getStream(this.sourceUrl); + const response = await plugins.smartrequest.SmartRequest.create() + .url(this.sourceUrl) + .get(); + const webStream = response.stream(); + // @ts-ignore - Web stream to Node.js stream conversion + const urlStream = plugins.stream.Readable.fromWeb(webStream); return urlStream; } if (this.sourceFilePath) { diff --git a/ts/classes.ziptools.ts b/ts/classes.ziptools.ts index 1d4c326..be9739e 100644 --- a/ts/classes.ziptools.ts +++ b/ts/classes.ziptools.ts @@ -19,9 +19,9 @@ class DecompressZipTransform extends plugins.smartstream.SmartDuplex { + writeFunction: async (chunkArg, streamtoolsArg) => { this.streamtools? null : this.streamtools = streamtoolsArg; - this.unzipper.push(chunkArg, false); + this.unzipper.push(Buffer.isBuffer(chunkArg) ? chunkArg : Buffer.from(chunkArg), false); }, finalFunction: async () => { this.unzipper.push(Buffer.from(''), true);