From 46269416137d8773a931cd58b3ee7facdf945607 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Mon, 18 Aug 2025 02:49:22 +0000 Subject: [PATCH] feat(MongoDumpTarget): Implement core MongoDumpTarget methods and update documentation & project configs --- .../document_symbols_cache_v23-06-25.pkl | Bin 0 -> 28567 bytes .serena/project.yml | 68 +++++ changelog.md | 8 + readme.md | 261 +++++++++++++++--- ts/00_commitinfo_data.ts | 2 +- 5 files changed, 299 insertions(+), 40 deletions(-) create mode 100644 .serena/cache/typescript/document_symbols_cache_v23-06-25.pkl create mode 100644 .serena/project.yml diff --git a/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl b/.serena/cache/typescript/document_symbols_cache_v23-06-25.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3498ce8407bb70f0a35f2bccdfee3561d7301386 GIT binary patch literal 28567 zcmeHQZ)_Y#6;B(-PVCx_lQvDO5H>We?bNyRza);6v~uhO1rx`LlO_U%cyD(udzgj? zo=3b)+>#h{WO(%O#Mt=J@!XN&Tz+Eo=)|GCId0}h%+W(6ryqhL2XIDhC4C7dvF$7l zI|IX3EA1vJ>xE*qvZ-2WyAIQ>&Tgq}arM$QQpGM?yt3|=gJ0Ww(#Oe81A;ks>k51*ddA9Ld^2Nj3vXw{Pu=YrCMsR>LY3 zO_!9|RIRQ`m*JGWvg;dV$E{l3R*%&?yUp5;{n$|LFx0)#P*~gH`_}HJ6nn0~doSzo zv~tzztZXehhK_R$-g2|=7OIu`e4$7tv)SxYshlwHEmtFQfE8{xG^#nZG zrdU-&rdnF%K0W6-MZZku^|Dp1oVkk+xz8m`; z<&&#aNQh^+h2;bU?HmMKIS6{7wpoAxDS9K2u$BP|fPg->HxOu)Mf%(j=^pMeUC^yF z8U$-|!eeSF9-@!2`mGTQrQbS`0;<-Kb=*35HzBmL*0AsEFzwD%r#+U&v?-?d2bfYk z&*f||1ZVZ-5ZDe5*fxMNR$;826@bOIZ4vZXdtU@>x*VEKk3(%<*@-y9Yt(j*Blkm1 z)F`BgH41AxDMu*x`(@nEVd0su<4hbyq!{2%;*ih@S)LId^xQn#8!-W*%`&OMRM^O1I$>y-cP`=9y9fd_Bs} zc8;AL96P%t>_m!P5yP?e2rwMji7a2K)Os-VF+JR4wn8`XU>Isxn-);3r9cmcqV}Q! z)V`h$YSs(Xn^7TnUrO&4DAo_+UTmsZoOX(Af8$+nW(u%Bk%`tNKsnY_18SnCA_Xkv zOe(PU7{xqrSXU~YM)aWY5xiI}asyxug(hhX7EKEy_O!62)Jb`bnlh)UV-Y~ewprF(A_B)Vh1gtZxgLA4ZpDj8(obI+ztY4d2y4ZY}-P=vZUm-966WNYMHU=97& zq;F75(XEt2-Kla&)ynrq+92;0ZFjutorW-Vh(5pOod>+*HSZ$S#5E5oVl$4lgAw`) zJO+DDs_0XvL95TYMh?R~$2_cE5mX1;V&-8jtCV>I3YgaeQJ2_d@Z@rqrLMu0i7TZOd=L2zp+ zqOFc9R-H+eRf^2MJ)Y`YQ1jJRs2jK14cNx^{Q%TNy+evv@34mF1hL-j$QMdFgfdDc zdQ^MSEru%2`|1xt*I20bLG8F8R6GUNAiPPpPxe!4^iNj+NB5F`^Nd*}7Ya+{>_W(_ z4$kX(6kfhP^185w+m@u)RZGEevHDdpYe@mKx`0{pj#FGHJRP|xJgfF|til?k0&uLV zrHEX3RjgV~l~t;Y)#X&*f*Pw1dX%4&>&13}H?~(;dsF}$+u)>^1w*kGZ)jEcqn}a0 zqi)8d>0%+sKA~HGUM+e#PGN0a;1ssSoWdF^j6yJPDO0_OunPunY{fIc%|8G_D}Ig zoqR<5$M7M{iPCmZ+HT+enWR}0&g#SM8F~$8;T+w54W5}2ImB()b z=&Fwrs9&ktufn9LpbXXTkKz#>#~-}lLWr`W{TC%|*nfeym@U|E;*WX)@+bHZ1tg$t zr?em-Z%LYE$sG3YfRNAUuzwFtR*(G#`~dO%0~?8ZP#=BJy~2Zb(q`D>*>A6}+h3s@ zrhwSLgqL-BTh1|8tCeRH$;1QH2a7z5Dfil>wp{eD6-pl1v>?+u zgE_xB-6NSZfvuW}s0X!Mof`ZrTLqSl9e<{s&}zp%2IOm1%(A5tAta@6GSM!Vi4M6; zbTyF))@_#{N~{?U18gc2(bQ^ES-8pd$S1dz&Z1Lz3TvLOC$NpXL4c1<4g~gJY6N!M zy=e)o{SSBp-~K0*l}RcgU?3ri4*^z?nG#visFX#m5*ZylR>ibnRqDG*2{+ze_H)I8 zp}{SKGW^iJ)smrcSz8iLtV3G^^=HZ%G5I!FAE>;(q8G~IAQ=qN4Gf}|Ynn^SzFQJW zL<>+ii-=RVI=0}F+!h#3Yys=mD`+WehO8r|ZWt)1b}E^B#7(S@9ay9@5TC%BQK6*^ zSMzSs!nGSkGW|&YBT6KHPn;y7Ie9Ivd9cDHmJ2TC-wF;_<@T60Lt2X*hsNFDj3Bnh zif|ZKg2RCX=fNOQ9S;_F{`3n}yy`=^PL=(ASjF34#$6yY!PNGm=PpTb90<%CBxD56 z5?sqFl&%d0ZFr^Zv|kD?d%=fKPHF?DV+3S+X?0LL@dOz6nA|85y#!f7Kmgp%*dPa|qm;xT(F(>1Qxu0ZI>8}LU|eFxNpG>WtT)as)LA%) zeBrQYGb9Jd?E>p2nswHU<|MMuiZ&KgDm0_=TvN4(voL5X0o9Qh>OcoVkqx{)Gna`a zgQ^coF(x?$#8gqJJ|u-en!MKv<|pKs&(Y?(n@QG9#GExlvR{p}Af)_pbF1SoOw&n- z`v%qw*?@9+Y}^gb+G2V9FbYcg{=y|C@^~f|EHHCmJ44A@2I5`gvu39JAQb-bz4WJ=^F z?x~eJ+;b}4_wsk(DK(x&>=aOUk$J75&)AY9LEXwp&e=v6^Fk=(p_2~0Mh|^b3*B?5 zWUdxW>UOY4quYB2s*@Ncwth2rwrmAsQYi_SKFK}+9+L33p~n1b=E(;c_w{UY@Ps0S zKr0n_^$CX&vA4}Tikz3iZh$tTE_OA(&At+Z`f(kK$X_~9dYlJbV8eDA#=vD&K~4E zHICxp#zop(cMrn4iT)^ShOFh-I4atMd_je=s65@!QpEcY%mPnu_t89ux)tc><lxdGCW{4FtE5tkDc-FhnUy0sL7VGtDp@xX zN!AQ0;S9<3N1(^(48$j}=DHE+=G~$Q$2SrC^mmlsQzBxiMW9D04&u(9HA6zWT-6(Q z<08kNLdh|WzCStUSq{&*9^D` z_Q*HV56P{_KH3a75ad>*PB+#Ji8`^v@N+fR{5MKeXdgVK_Cg2PlY}LSl33{12+ri- z%10nrnwD|Na#1(PK>zWJ3#ZP{O`SdUME_9Z&K_~YZAGHH56!qCS!Z{A`Z~Mflh@gG zerjRi(z)|9&AQLSw=hcHc^R5l9M@cD_nDKe@H?;vwQ!laR=7=#Ng|{r+-2c3vO}(s zk5biZ7eZUC8AI>V&{pGavB`k5RYe;4rV5R$?BH$-O==^<+ z%emjU8yERi#Qk@baKA5c{RnxF%vX4P{|gl_dHEZgg=iv8Mw-;-sXufaVth|9--&uf z8%xB-5+UTMhRy(8BB-aCa2j&-u2Qr!2?LVhiFS~=NNK=%4K9-o)JoY9lR{0a(Mz_Z z)mqEogEi({CuhWaHEetRea>3T0t(>W7UD?3`4_N(o`)oiS_(CrTh z`Rxc?l5mPQ2yE8i9g^{u=#E?~7Joan_+qq_HN*8exy5InZxfbj*36x_DvFl2ftgAt z6+D?)`v?n}QSr{jS)!q&&g7D3-@;YARsiXh1Hzi&mYy6C_PGcMYsQs33NR@Gq)!PT zqdXwgM%2QRFd#{MYAyh_f>4hfA=X^?c$|GMBE*{E!k}`5_9{W>B!^IBGGmOIUWZ5` zCR{^o1+Q*7UaT2zD9Sa8eJT1NHqfCf)~S@xt_D z)_;T$Ug|df@=M)tSMlbu<3d&@{zU}w-bCPKFed%)-zh*}Z9NhZccW|Nx)fg=gt=j{ z?Fg6MqMmnL7uXBQO9?&jObXIh#Fj07~ru&IP|B#X|?N z3_;<@Ni17>SzFOP&08i$AzuI=(u6-B0pVh}N>u#3Fv4t7HunSWX^45&fQVw97;I1 z){EjAr*Ew%hvzniPa!)@`t*%!V_', - mongoDbUser: '', - mongoDbPass: '', - mongoDbUrl: 'mongodb+srv://:@/?retryWrites=true&w=majority', +const mongoDescriptor: IMongoDescriptor = { + mongoDbName: 'production_db', + mongoDbUser: process.env.MONGO_USER, + mongoDbPass: process.env.MONGO_PASS, + mongoDbUrl: process.env.MONGO_URL, }; ``` -### Creating and Using the MongoDump Instance -To interact with MongoDB for dumping purposes, you'll use the `MongoDump` class: +### Working with MongoDump + +The `MongoDump` class is your main entry point for managing database dumps: ```typescript -async function setupMongoDump() { - const mongoDump = new MongoDump(); - const mongoDumpTarget = await mongoDump.addMongoTargetByMongoDescriptor(myMongoDescriptor); +const mongoDump = new MongoDump(); - // mongoDumpTarget can now be used for further operations -} -setupMongoDump(); +// Add a MongoDB target +const dumpTarget = await mongoDump.addMongoTargetByMongoDescriptor(mongoDescriptor); + +// The target is now ready for dump operations ``` -### Dumping Collections to a Directory -To dump the collections of a database into a directory with files representing each document, you can use the `dumpCollectionToDir` method. This method is useful for creating backups or migrating data: +### Dumping Collections + +#### Dump All Collections to Directory + +Perfect for complete database backups: ```typescript -async function dumpCollections() { - const mongoDump = new MongoDump(); - const mongoDumpTarget = await mongoDump.addMongoTargetByMongoDescriptor(myMongoDescriptor); +// Basic dump - uses document _id as filename +await dumpTarget.dumpAllCollectionsToDir('./backups/full-backup'); - await mongoDumpTarget.dumpAllCollectionsToDir('./path/to/dumpDir', null, true); - // This dumps all collections to the specified directory, cleaning the directory before dumping. -} -dumpCollections(); +// Custom naming - use any document field for filenames +await dumpTarget.dumpAllCollectionsToDir( + './backups/named-backup', + (doc) => `${doc.username}_${doc.timestamp}`, + true // Clean directory before dumping +); ``` -### Advanced Dumping Options -For more control over the dumping process, including naming conventions for dumped files or handling specific collections, you can explore methods like `dumpCollectionToDir` for individual collections and advanced configurations concerning directory cleanliness and document naming. +#### Dump Specific Collection -### Shutting Down Properly -It's important to close down database connections properly once your dumping operations are complete: +When you need granular control: ```typescript -async function shutDownMongoDump() { +// Get available collections +const collections = await dumpTarget.getCollections(); +console.log('Available collections:', collections); + +// Dump a specific collection +const userCollection = collections.find(c => c.collectionName === 'users'); +await dumpTarget.dumpCollectionToDir( + userCollection, + './backups/users', + (doc) => doc.email.replace('@', '_at_') // Custom naming +); +``` + +### Archive Support + +Create compressed archives for efficient storage and transfer: + +```typescript +// Dump to tar archive file +await dumpTarget.dumpCollectionToTarArchiveFile( + userCollection, + './backups/users.tar.gz' +); + +// Or work with streams for advanced scenarios +const archiveStream = await dumpTarget.dumpAllCollectionsToTarArchiveStream(); +// Pipe to S3, network storage, etc. +``` + +### Advanced Patterns + +#### Scheduled Backups + +Implement automated backup strategies: + +```typescript +import { CronJob } from 'cron'; + +const backupJob = new CronJob('0 0 * * *', async () => { + const timestamp = new Date().toISOString().split('T')[0]; const mongoDump = new MongoDump(); + const target = await mongoDump.addMongoTargetByMongoDescriptor(mongoDescriptor); + + await target.dumpAllCollectionsToDir( + `./backups/daily/${timestamp}`, + null, + true + ); + await mongoDump.stop(); - // Closes all open database connections gracefully -} -shutDownMongoDump(); + console.log(`✅ Daily backup completed: ${timestamp}`); +}); + +backupJob.start(); ``` -### Conclusion -The @push.rocks/mongodump module provides a flexible approach to handling MongoDB database dumps, whether it's for backup, migration, or other purposes. By leveraging TypeScript and modern async patterns, it integrates smoothly into modern Node.js applications focused on MongoDB interactions. +#### Multiple Database Targets +Handle multiple databases simultaneously: + +```typescript +const mongoDump = new MongoDump(); + +// Add multiple targets +const prodTarget = await mongoDump.addMongoTargetByMongoDescriptor(prodDescriptor); +const stagingTarget = await mongoDump.addMongoTargetByMongoDescriptor(stagingDescriptor); + +// Dump both in parallel +await Promise.all([ + prodTarget.dumpAllCollectionsToDir('./backups/production'), + stagingTarget.dumpAllCollectionsToDir('./backups/staging') +]); + +await mongoDump.stop(); +``` + +#### Error Handling + +Implement robust error handling: + +```typescript +try { + const mongoDump = new MongoDump(); + const target = await mongoDump.addMongoTargetByMongoDescriptor(mongoDescriptor); + + await target.dumpAllCollectionsToDir('./backups/safe-backup'); + console.log('✅ Backup successful'); + +} catch (error) { + console.error('❌ Backup failed:', error); + // Implement notification/retry logic + +} finally { + await mongoDump.stop(); +} +``` + +### Clean Shutdown + +Always ensure proper cleanup of database connections: + +```typescript +// Closes all open MongoDB connections +await mongoDump.stop(); +``` + +## API Reference 🔧 + +### MongoDump + +The main class for managing dump operations. + +**Methods:** +- `addMongoTargetByMongoDescriptor(descriptor: IMongoDescriptor)`: Create a new dump target +- `stop()`: Close all database connections + +### MongoDumpTarget + +Handles operations for a specific MongoDB database. + +**Methods:** +- `getCollections()`: List all collections in the database +- `dumpCollectionToDir(collection, directory, nameTransform?)`: Dump single collection +- `dumpAllCollectionsToDir(directory, nameTransform?, cleanDir?)`: Dump all collections +- `dumpCollectionToTarArchiveFile(collection, filepath)`: Create tar archive +- `dumpAllCollectionsToTarArchiveStream()`: Get tar archive stream + +## Best Practices 💡 + +1. **Environment Variables**: Store credentials in environment variables, never hardcode them +2. **Error Handling**: Always wrap dump operations in try-catch blocks +3. **Connection Cleanup**: Always call `stop()` when done +4. **Backup Verification**: Periodically verify your backups can be restored +5. **Storage Rotation**: Implement backup rotation to manage disk space +6. **Monitoring**: Add logging and monitoring for production backup jobs + +## Why Choose @push.rocks/mongodump? 🌟 + +- **Type Safety**: Full TypeScript support with comprehensive type definitions +- **Modern Async**: Built on promises and async/await patterns +- **Flexible Export**: Multiple export formats and naming options +- **Production Ready**: Battle-tested in production environments +- **Clean API**: Intuitive, well-documented interface +- **Active Maintenance**: Regular updates and security patches ## License and Legal Information This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 6452858..7e14f48 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/mongodump', - version: '1.0.10', + version: '1.1.0', description: 'A tool to create and manage dumps of MongoDB databases, supporting data export and import.' }