v7.1.5

changelog.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2026-04-05 - 7.1.5 - fix(collection)
+ensure unique indexes are marked before upsert operations
+
+- calls unique index marking during collection updates before executing upsert logic
+- helps keep update behavior aligned with index handling already applied on inserts
+
+## 2026-04-05 - 7.1.4 - fix(collection)
+improve index creation resilience and add collection integrity checks
+
+- Handle MongoDB index creation failures with structured logging instead of failing silently or racing on repeated attempts
+- Log duplicate field values when unique index creation fails due to existing duplicate data
+- Await unique and regular index creation during insert operations to ensure index setup completes predictably
+- Add collection integrity checks for estimated vs actual document counts and duplicate values on tracked unique fields
+- Expose collection integrity checks through the document class API
+
+## 2026-03-26 - 7.1.3 - fix(deps)
+bump development dependencies for tooling and Node types
+
+- update @git.zone/tsrun from ^2.0.1 to ^2.0.2
+- update @git.zone/tstest from ^3.5.1 to ^3.6.0
+- update @types/node from ^22.15.2 to ^25.5.0
+
 ## 2026-03-24 - 7.1.2 - fix(docs)
 refresh project guidance for TC39 decorators, build configuration, and dependency compatibility
 

deno.lock

@@ -2,8 +2,8 @@
   "version": "5",
   "specifiers": {
     "npm:@git.zone/tsbuild@^4.4.0": "4.4.0",
-    "npm:@git.zone/tsrun@^2.0.1": "2.0.1",
-    "npm:@git.zone/tstest@^3.5.1": "3.5.1",
+    "npm:@git.zone/tsrun@^2.0.2": "2.0.2",
+    "npm:@git.zone/tstest@^3.6.0": "3.6.0",
     "npm:@push.rocks/lik@^6.4.0": "6.4.0",
     "npm:@push.rocks/qenv@^6.1.3": "6.1.3",
     "npm:@push.rocks/smartdelay@^3.0.5": "3.0.5",
@@ -16,7 +16,7 @@
     "npm:@push.rocks/smartunique@^3.0.9": "3.0.9",
     "npm:@push.rocks/taskbuffer@^8.0.2": "8.0.2",
     "npm:@tsclass/tsclass@^9.5.0": "9.5.0",
-    "npm:@types/node@^22.15.2": "22.19.1",
+    "npm:@types/node@^25.5.0": "25.5.0",
     "npm:mongodb@^7.1.1": "7.1.1"
   },
   "npm": {
@@ -903,18 +903,18 @@
       "bin": true,
       "tarball": "https://verdaccio.lossless.digital/@git.zone/tspublish/-/tspublish-1.11.4.tgz"
     },
-    "@git.zone/tsrun@2.0.1": {
-      "integrity": "sha512-NEcnsjvlC1o3Z6SS3VhKCf6Ev+Sh4EAinmggslrIR/ppMrvjDbXNFXoyr3PB+GLeSAR0JRZ1fGvVYjpEzjBdIg==",
+    "@git.zone/tsrun@2.0.2": {
+      "integrity": "sha512-Rnp/wYHzI8A1pVBKOOePRJgQiBZdW+GEjpQk2uhvXz6A+ljUV2SXKc7NpQVVDsjEZaNFeAI9jMYOdk3lm3yMDA==",
       "dependencies": [
         "@push.rocks/smartfile@13.1.2",
         "@push.rocks/smartshell",
         "tsx"
       ],
       "bin": true,
-      "tarball": "https://verdaccio.lossless.digital/@git.zone/tsrun/-/tsrun-2.0.1.tgz"
+      "tarball": "https://verdaccio.lossless.digital/@git.zone/tsrun/-/tsrun-2.0.2.tgz"
     },
-    "@git.zone/tstest@3.5.1": {
-      "integrity": "sha512-R1T3Tr9zun0B5bFq8qK8/KZ2uqHZHfDPB3/mhoL5ekwxGSI4InX5sXMpbNuhw2EdNZ7rMAHNCyeSLz2PwFmNZw==",
+    "@git.zone/tstest@3.6.0": {
+      "integrity": "sha512-5D6COywCXmCqeUB8v6/kOzjEWCTKTUTI3ZB99ebwEibENFXnFBoVxNSRKN0pSmBYlgBEkT7DLNfTfp5tclSg8A==",
       "dependencies": [
         "@git.zone/tsbundle",
         "@git.zone/tsrun",
@@ -944,7 +944,7 @@
         "ws"
       ],
       "bin": true,
-      "tarball": "https://verdaccio.lossless.digital/@git.zone/tstest/-/tstest-3.5.1.tgz"
+      "tarball": "https://verdaccio.lossless.digital/@git.zone/tstest/-/tstest-3.6.0.tgz"
     },
     "@img/colour@1.1.0": {
       "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==",
@@ -1151,7 +1151,7 @@
         "@inquirer/figures",
         "@inquirer/type",
         "@types/mute-stream",
-        "@types/node@22.19.1",
+        "@types/node@22.19.15",
         "@types/wrap-ansi",
         "ansi-escapes",
         "cli-width",
@@ -3486,12 +3486,12 @@
       "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==",
       "tarball": "https://verdaccio.lossless.digital/@types/node/-/node-16.9.1.tgz"
     },
-    "@types/node@22.19.1": {
-      "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==",
+    "@types/node@22.19.15": {
+      "integrity": "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==",
       "dependencies": [
         "undici-types@6.21.0"
       ],
-      "tarball": "https://verdaccio.lossless.digital/@types/node/-/node-22.19.1.tgz"
+      "tarball": "https://verdaccio.lossless.digital/@types/node/-/node-22.19.15.tgz"
     },
     "@types/node@24.2.0": {
       "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==",
@@ -3500,6 +3500,13 @@
       ],
       "tarball": "https://verdaccio.lossless.digital/@types/node/-/node-24.2.0.tgz"
     },
+    "@types/node@25.5.0": {
+      "integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==",
+      "dependencies": [
+        "undici-types@7.18.2"
+      ],
+      "tarball": "https://verdaccio.lossless.digital/@types/node/-/node-25.5.0.tgz"
+    },
     "@types/ping@0.4.4": {
       "integrity": "sha512-ifvo6w2f5eJYlXm+HiVx67iJe8WZp87sfa683nlqED5Vnt9Z93onkokNoWqOG21EaE8fMxyKPobE+mkPEyxsdw==",
       "tarball": "https://verdaccio.lossless.digital/@types/ping/-/ping-0.4.4.tgz"
@@ -5573,9 +5580,9 @@
       ],
       "tarball": "https://verdaccio.lossless.digital/no-case/-/no-case-2.3.2.tgz"
     },
-    "node-forge@1.3.3": {
-      "integrity": "sha512-rLvcdSyRCyouf6jcOIPe/BgwG/d7hKjzMKOas33/pHEr6gbq18IK9zV7DiPvzsz0oBJPme6qr6H6kGZuI9/DZg==",
-      "tarball": "https://verdaccio.lossless.digital/node-forge/-/node-forge-1.3.3.tgz"
+    "node-forge@1.4.0": {
+      "integrity": "sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==",
+      "tarball": "https://verdaccio.lossless.digital/node-forge/-/node-forge-1.4.0.tgz"
     },
     "normalize-url@8.1.1": {
       "integrity": "sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==",
@@ -6542,6 +6549,10 @@
       "integrity": "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==",
       "tarball": "https://verdaccio.lossless.digital/undici-types/-/undici-types-7.10.0.tgz"
     },
+    "undici-types@7.18.2": {
+      "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
+      "tarball": "https://verdaccio.lossless.digital/undici-types/-/undici-types-7.18.2.tgz"
+    },
     "unified@11.0.5": {
       "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==",
       "dependencies": [
@@ -6778,8 +6789,8 @@
     "packageJson": {
       "dependencies": [
         "npm:@git.zone/tsbuild@^4.4.0",
-        "npm:@git.zone/tsrun@^2.0.1",
-        "npm:@git.zone/tstest@^3.5.1",
+        "npm:@git.zone/tsrun@^2.0.2",
+        "npm:@git.zone/tstest@^3.6.0",
         "npm:@push.rocks/lik@^6.4.0",
         "npm:@push.rocks/qenv@^6.1.3",
         "npm:@push.rocks/smartdelay@^3.0.5",
@@ -6792,7 +6803,7 @@
         "npm:@push.rocks/smartunique@^3.0.9",
         "npm:@push.rocks/taskbuffer@^8.0.2",
         "npm:@tsclass/tsclass@^9.5.0",
-        "npm:@types/node@^22.15.2",
+        "npm:@types/node@^25.5.0",
         "npm:mongodb@^7.1.1"
       ]
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "7.1.2",
+  "version": "7.1.5",
   "private": false,
   "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
   "exports": {
@@ -39,10 +39,10 @@
   },
   "devDependencies": {
     "@git.zone/tsbuild": "^4.4.0",
-    "@git.zone/tsrun": "^2.0.1",
-    "@git.zone/tstest": "^3.5.1",
+    "@git.zone/tsrun": "^2.0.2",
+    "@git.zone/tstest": "^3.6.0",
     "@push.rocks/qenv": "^6.1.3",
-    "@types/node": "^22.15.2"
+    "@types/node": "^25.5.0"
   },
   "files": [
     "ts/**/*",

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '7.1.2',
+  version: '7.1.5',
   description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/classes.collection.ts

@@ -216,8 +216,15 @@ export class SmartdataCollection<T> {
         const indexSpec: Record<string, 'text'> = {};
         searchableFields.forEach(f => { indexSpec[f] = 'text'; });
         // Cast to any to satisfy TypeScript IndexSpecification typing
-        await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
-        this.textIndexCreated = true;
+        try {
+          await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
+          this.textIndexCreated = true;
+        } catch (err: any) {
+          logger.log(
+            'warn',
+            `Failed to create text index on fields [${searchableFields.join(', ')}] in collection "${this.collectionName}": ${err?.message || String(err)}`
+          );
+        }
       }
     }
   }
@@ -228,11 +235,25 @@ export class SmartdataCollection<T> {
   public async markUniqueIndexes(keyArrayArg: string[] = []) {
     for (const key of keyArrayArg) {
       if (!this.uniqueIndexes.includes(key)) {
-        await this.mongoDbCollection.createIndex({ [key]: 1 }, {
-          unique: true,
-        });
-        // make sure we only call this once and not for every doc we create
+        // Claim the slot immediately to prevent concurrent inserts from retrying
         this.uniqueIndexes.push(key);
+        try {
+          await this.mongoDbCollection.createIndex({ [key]: 1 }, {
+            unique: true,
+          });
+        } catch (err: any) {
+          const errorCode = err?.code || err?.codeName || 'unknown';
+          const errorMessage = err?.message || String(err);
+          logger.log(
+            'error',
+            `Failed to create unique index on field "${key}" in collection "${this.collectionName}". ` +
+            `MongoDB error [${errorCode}]: ${errorMessage}. ` +
+            `Uniqueness constraint on "${key}" is NOT enforced.`
+          );
+          if (errorCode === 11000 || errorCode === 'DuplicateKey' || String(errorMessage).includes('E11000')) {
+            await this.logDuplicatesForField(key);
+          }
+        }
       }
     }
   }
@@ -245,16 +266,66 @@ export class SmartdataCollection<T> {
       // Check if we've already created this index
       const indexKey = indexDef.field;
       if (!this.regularIndexes.some(i => i.field === indexKey)) {
-        await this.mongoDbCollection.createIndex(
-          { [indexDef.field]: 1 }, // Simple single-field index
-          indexDef.options
-        );
-        // Track that we've created this index
+        // Claim the slot immediately to prevent concurrent retries
         this.regularIndexes.push(indexDef);
+        try {
+          await this.mongoDbCollection.createIndex(
+            { [indexDef.field]: 1 }, // Simple single-field index
+            indexDef.options
+          );
+        } catch (err: any) {
+          const errorCode = err?.code || err?.codeName || 'unknown';
+          const errorMessage = err?.message || String(err);
+          logger.log(
+            'warn',
+            `Failed to create index on field "${indexKey}" in collection "${this.collectionName}". ` +
+            `MongoDB error [${errorCode}]: ${errorMessage}.`
+          );
+          if (
+            indexDef.options?.unique &&
+            (errorCode === 11000 || errorCode === 'DuplicateKey' || String(errorMessage).includes('E11000'))
+          ) {
+            await this.logDuplicatesForField(indexKey);
+          }
+        }
       }
     }
   }
 
+  /**
+   * Logs duplicate values for a field to help diagnose unique index creation failures.
+   */
+  private async logDuplicatesForField(field: string): Promise<void> {
+    try {
+      const pipeline = [
+        { $group: { _id: `$${field}`, count: { $sum: 1 }, ids: { $push: '$_id' } } },
+        { $match: { count: { $gt: 1 } } },
+        { $limit: 5 },
+      ];
+      const duplicates = await this.mongoDbCollection.aggregate(pipeline).toArray();
+      if (duplicates.length > 0) {
+        for (const dup of duplicates) {
+          logger.log(
+            'warn',
+            `Duplicate values for "${field}" in "${this.collectionName}": ` +
+            `value=${JSON.stringify(dup._id)} appears ${dup.count} times ` +
+            `(document _ids: ${JSON.stringify(dup.ids.slice(0, 5))})`
+          );
+        }
+        logger.log(
+          'warn',
+          `Unique index on "${field}" in "${this.collectionName}" was NOT created. ` +
+          `Resolve duplicates and restart to enforce uniqueness.`
+        );
+      }
+    } catch (aggErr: any) {
+      logger.log(
+        'warn',
+        `Could not identify duplicate documents for field "${field}" in "${this.collectionName}": ${aggErr?.message || String(aggErr)}`
+      );
+    }
+  }
+
   /**
    * adds a validation function that all newly inserted and updated objects have to pass
    */
@@ -295,6 +366,28 @@ export class SmartdataCollection<T> {
     const cursor = this.mongoDbCollection.find(filterObject, { session: opts?.session });
     const result = await cursor.toArray();
     cursor.close();
+
+    // In-memory check for duplicate _id values (should never happen)
+    if (result.length > 0) {
+      const idSet = new Set<string>();
+      const duplicateIds: string[] = [];
+      for (const doc of result) {
+        const idStr = String(doc._id);
+        if (idSet.has(idStr)) {
+          duplicateIds.push(idStr);
+        } else {
+          idSet.add(idStr);
+        }
+      }
+      if (duplicateIds.length > 0) {
+        logger.log(
+          'error',
+          `Integrity issue in "${this.collectionName}": found ${duplicateIds.length} duplicate _id values ` +
+          `in findAll results: [${duplicateIds.slice(0, 5).join(', ')}]. This should never happen.`
+        );
+      }
+    }
+
     return result;
   }
 
@@ -346,11 +439,11 @@ export class SmartdataCollection<T> {
   ): Promise<any> {
     await this.init();
     await this.checkDoc(dbDocArg);
-    this.markUniqueIndexes(dbDocArg.uniqueIndexes);
-    
+    await this.markUniqueIndexes(dbDocArg.uniqueIndexes);
+
     // Create regular indexes if available
     if (dbDocArg.regularIndexes && dbDocArg.regularIndexes.length > 0) {
-      this.createRegularIndexes(dbDocArg.regularIndexes);
+      await this.createRegularIndexes(dbDocArg.regularIndexes);
     }
     
     const saveableObject = await dbDocArg.createSavableObject() as any;
@@ -367,6 +460,7 @@ export class SmartdataCollection<T> {
   ): Promise<any> {
     await this.init();
     await this.checkDoc(dbDocArg);
+    await this.markUniqueIndexes(dbDocArg.uniqueIndexes);
     const identifiableObject = await dbDocArg.createIdentifiableObject();
     const saveableObject = await dbDocArg.createSavableObject() as any;
     const updateableObject: any = {};
@@ -402,6 +496,74 @@ export class SmartdataCollection<T> {
     return this.mongoDbCollection.countDocuments(filterObject, { session: opts?.session });
   }
 
+  /**
+   * Runs an integrity check on the collection.
+   * Compares estimated vs actual document count and checks for duplicates on unique index fields.
+   */
+  public async checkCollectionIntegrity(): Promise<{
+    ok: boolean;
+    estimatedCount: number;
+    actualCount: number;
+    duplicateFields: Array<{ field: string; duplicateValues: number }>;
+  }> {
+    await this.init();
+    const result = {
+      ok: true,
+      estimatedCount: 0,
+      actualCount: 0,
+      duplicateFields: [] as Array<{ field: string; duplicateValues: number }>,
+    };
+
+    try {
+      result.estimatedCount = await this.mongoDbCollection.estimatedDocumentCount();
+      result.actualCount = await this.mongoDbCollection.countDocuments({});
+
+      if (result.estimatedCount !== result.actualCount) {
+        result.ok = false;
+        logger.log(
+          'warn',
+          `Integrity check on "${this.collectionName}": estimatedDocumentCount=${result.estimatedCount} ` +
+          `but countDocuments=${result.actualCount}. Possible data inconsistency.`
+        );
+      }
+
+      // Check for duplicates on each tracked unique index field
+      for (const field of this.uniqueIndexes) {
+        try {
+          const pipeline = [
+            { $group: { _id: `$${field}`, count: { $sum: 1 } } },
+            { $match: { count: { $gt: 1 } } },
+            { $count: 'total' },
+          ];
+          const countResult = await this.mongoDbCollection.aggregate(pipeline).toArray();
+          const dupCount = countResult[0]?.total || 0;
+          if (dupCount > 0) {
+            result.ok = false;
+            result.duplicateFields.push({ field, duplicateValues: dupCount });
+            logger.log(
+              'warn',
+              `Integrity check on "${this.collectionName}": field "${field}" has ${dupCount} values with duplicates ` +
+              `despite being marked as unique.`
+            );
+          }
+        } catch (fieldErr: any) {
+          logger.log(
+            'warn',
+            `Integrity check: could not verify uniqueness of "${field}" in "${this.collectionName}": ${fieldErr?.message || String(fieldErr)}`
+          );
+        }
+      }
+    } catch (err: any) {
+      result.ok = false;
+      logger.log(
+        'error',
+        `Integrity check failed for "${this.collectionName}": ${err?.message || String(err)}`
+      );
+    }
+
+    return result;
+  }
+
   /**
    * checks a Doc for constraints
    * if this.objectValidation is not set it passes.

ts/classes.doc.ts

@@ -597,6 +597,17 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     return await collection.getCount(filterArg);
   }
 
+  /**
+   * Runs an integrity check on this collection.
+   * Returns a summary with estimated vs actual counts and any duplicate unique fields.
+   */
+  public static async checkCollectionIntegrity<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    return await collection.checkCollectionIntegrity();
+  }
+
   /**
    * Create a MongoDB filter from a Lucene query string
    * @param luceneQuery Lucene query string