v7.1.5

changelog.md

@@ -1,5 +1,34 @@
 # Changelog
 
+## 2026-04-05 - 7.1.5 - fix(collection)
+ensure unique indexes are marked before upsert operations
+
+- calls unique index marking during collection updates before executing upsert logic
+- helps keep update behavior aligned with index handling already applied on inserts
+
+## 2026-04-05 - 7.1.4 - fix(collection)
+improve index creation resilience and add collection integrity checks
+
+- Handle MongoDB index creation failures with structured logging instead of failing silently or racing on repeated attempts
+- Log duplicate field values when unique index creation fails due to existing duplicate data
+- Await unique and regular index creation during insert operations to ensure index setup completes predictably
+- Add collection integrity checks for estimated vs actual document counts and duplicate values on tracked unique fields
+- Expose collection integrity checks through the document class API
+
+## 2026-03-26 - 7.1.3 - fix(deps)
+bump development dependencies for tooling and Node types
+
+- update @git.zone/tsrun from ^2.0.1 to ^2.0.2
+- update @git.zone/tstest from ^3.5.1 to ^3.6.0
+- update @types/node from ^22.15.2 to ^25.5.0
+
+## 2026-03-24 - 7.1.2 - fix(docs)
+refresh project guidance for TC39 decorators, build configuration, and dependency compatibility
+
+- streamlines readme hints to focus on current decorator patterns and runtime support
+- adds compatibility notes for the updated build toolchain and dependency APIs
+- includes the project license file in the repository
+
 ## 2026-03-24 - 7.1.1 - fix(build)
 update build and test tooling configuration, migrate project config to .smartconfig.json, and align TypeScript typings
 

license

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Task Venture Capital GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartdata",
-  "version": "7.1.1",
+  "version": "7.1.5",
   "private": false,
   "description": "An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.",
   "exports": {
@@ -39,10 +39,10 @@
   },
   "devDependencies": {
     "@git.zone/tsbuild": "^4.4.0",
-    "@git.zone/tsrun": "^2.0.1",
-    "@git.zone/tstest": "^3.5.1",
+    "@git.zone/tsrun": "^2.0.2",
+    "@git.zone/tstest": "^3.6.0",
     "@push.rocks/qenv": "^6.1.3",
-    "@types/node": "^22.15.2"
+    "@types/node": "^25.5.0"
   },
   "files": [
     "ts/**/*",

readme.hints.md

@@ -1,113 +1,29 @@
 # Project Memory - Smartdata
 
-## TC39 Decorator Migration (v6.0.0) - ✅ COMPLETED
-
-### Final Status: All Tests Passing (157/157)
-
-Migration successfully completed on 2025-11-17.
-
-### What Changed:
-
-- ✅ Removed `experimentalDecorators` from tsconfig.json
-- ✅ Refactored all 7 decorators to TC39 Stage 3 syntax
-  - 5 property decorators: @globalSvDb, @svDb, @unI, @index, @searchable
-  - 2 class decorators: @Collection, @managed
-- ✅ Implemented context.metadata pattern for shared decorator state
-- ✅ All tests passing across Node.js and Deno runtimes
-
-### Critical Discovery: TC39 Metadata Access Pattern
-
-**THE KEY INSIGHT**: In TC39 decorators, metadata is NOT accessed via `constructor[Symbol.metadata]`. Instead:
+## TC39 Decorator Pattern
 
 - **Field decorators**: Write to `context.metadata`
-- **Class decorators**: Read from `context.metadata` (same shared object!)
-- The `context.metadata` object is shared between all decorators on the same class
-- Attempting to write to `constructor[Symbol.metadata]` throws: "Cannot assign to read only property"
+- **Class decorators**: Read from `context.metadata` (same shared object)
+- `Symbol.metadata` on constructors is read-only (managed by runtime)
+- Field decorators run before class decorators (guaranteed order)
+- `declare` keyword for instance properties accessed via prototype getters (avoids ES2022 shadowing)
 
-### Implementation Pattern:
+### Runtime Compatibility
 
-```typescript
-// Field decorator - stores metadata
-export function svDb() {
-  return (value: undefined, context: ClassFieldDecoratorContext) => {
-    const metadata = context.metadata as ISmartdataDecoratorMetadata;
-    if (!metadata.saveableProperties) {
-      metadata.saveableProperties = [];
-    }
-    metadata.saveableProperties.push(String(context.name));
-  };
-}
+- ✅ Node.js v20+ / v25+: Full TC39 support
+- ✅ Deno v2.x: Full TC39 support
+- ❌ Bun: No TC39 support (uses legacy decorators only)
 
-// Class decorator - reads metadata and initializes prototype
-export function Collection(dbArg: SmartdataDb) {
-  return function(value: Function, context: ClassDecoratorContext) => {
-    const metadata = context.metadata as ISmartdataDecoratorMetadata;
-    if (metadata?.saveableProperties) {
-      decoratedClass.prototype.saveableProperties = [...metadata.saveableProperties];
-    }
-    return decoratedClass;
-  };
-}
-```
+## Build Configuration (v7.1.0+)
 
-### Runtime Compatibility:
+- **Build tool**: `@git.zone/tsbuild` v4 with `tsbuild tsfolders`
+- **tsconfig.json**: Includes `"types": ["node"]` since tsbuild v4 defaults to DOM+ESNext lib only
+- **Strict mode**: tsbuild v4 enables strict checks; properties use `!` definite assignment or `declare`
+- **Test imports**: Use `@git.zone/tstest/tapbundle` (NOT `@push.rocks/tapbundle`)
+- **Config file**: `.smartconfig.json` (renamed from `npmextra.json`)
 
-- ✅ **Node.js v23.8.0**: Full TC39 support
-- ✅ **Deno v2.5.4**: Full TC39 support
-- ❌ **Bun v1.3.0**: No TC39 support (uses legacy decorators only)
-  - Removed "+bun" from test filenames to skip Bun tests
+## Dependencies (v7.1.0+)
 
-### Key Technical Notes:
-
-1. **Metadata Initialization Timing**: Class decorators run AFTER field decorators, allowing them to read accumulated metadata and initialize prototypes before any instances are created
-2. **Prototype vs Instance Properties**: Properties set on prototype are accessible via `this.propertyName` in instances
-3. **TypeScript Lib Support**: TypeScript 5.9.3 includes built-in decorator types (no custom lib configuration needed)
-4. **Interface Naming**: Used `ISmartdataDecoratorMetadata` extending `DecoratorMetadataObject` for type safety
-
-### Files Modified:
-
-- ts/classes.doc.ts (property decorators + metadata interface)
-- ts/classes.collection.ts (class decorators + prototype initialization)
-- tsconfig.json (removed experimentalDecorators flag)
-- test/\*.ts (renamed files to remove "+bun" suffix)
-
-### Test Results:
-
-All 157 tests passing across 10 test files:
-
-- test.cursor.ts: 7/7
-- test.deno.ts: 11/11 (queries working correctly!)
-- test.search.advanced.ts: 41/41
-- test.typescript.ts: 4/4
-- test.watch.ts: 5/5
-- And 5 more test files
-
-### Migration Learnings for Future Reference:
-
-1. `context.metadata` is the ONLY way to share state between decorators
-2. Class decorators must initialize prototypes from metadata immediately
-3. `Symbol.metadata` on constructors is read-only (managed by runtime)
-4. Field decorators run before class decorators (guaranteed order)
-5. TypeScript 5.2+ has built-in TC39 decorator support
-
-## ES2022 Class Fields & Prototype Getters - Fixed in v7.0.15
-
-### Issue
-
-ES2022 class fields (`useDefineForClassFields: true`) create own properties during construction that shadow prototype getters defined by decorators.
-
-### Solution
-
-Use `declare` keyword for instance properties that are accessed via prototype getters:
-
-```typescript
-// In SmartDataDbDoc (ts/classes.doc.ts):
-declare public collection: SmartdataCollection<any>;  // Type-only, no JS emitted
-declare public manager: TManager;  // Type-only, no JS emitted
-```
-
-### Key Insight
-
-- `declare` tells TypeScript this is a type-only declaration
-- No JavaScript code is emitted for `declare` properties
-- Prototype getters defined by `@Collection` and `@managed` decorators are no longer shadowed
+- `@push.rocks/taskbuffer` v8: distributedCoordination API at `taskbuffer.distributedCoordination.*`
+- `@push.rocks/smartmongo` v5: API compatible (`createAndStart`, `getMongoDescriptor`, `stop`, `stopAndDumpToDir`)
+- `mongodb` v7.1: ChangeStream requires `Document` constraint, use `any` for generic watcher

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartdata',
-  version: '7.1.1',
+  version: '7.1.5',
   description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }

ts/classes.collection.ts

@@ -216,8 +216,15 @@ export class SmartdataCollection<T> {
         const indexSpec: Record<string, 'text'> = {};
         searchableFields.forEach(f => { indexSpec[f] = 'text'; });
         // Cast to any to satisfy TypeScript IndexSpecification typing
-        await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
-        this.textIndexCreated = true;
+        try {
+          await this.mongoDbCollection.createIndex(indexSpec as any, { name: 'smartdata_text_index' });
+          this.textIndexCreated = true;
+        } catch (err: any) {
+          logger.log(
+            'warn',
+            `Failed to create text index on fields [${searchableFields.join(', ')}] in collection "${this.collectionName}": ${err?.message || String(err)}`
+          );
+        }
       }
     }
   }
@@ -228,11 +235,25 @@ export class SmartdataCollection<T> {
   public async markUniqueIndexes(keyArrayArg: string[] = []) {
     for (const key of keyArrayArg) {
       if (!this.uniqueIndexes.includes(key)) {
-        await this.mongoDbCollection.createIndex({ [key]: 1 }, {
-          unique: true,
-        });
-        // make sure we only call this once and not for every doc we create
+        // Claim the slot immediately to prevent concurrent inserts from retrying
         this.uniqueIndexes.push(key);
+        try {
+          await this.mongoDbCollection.createIndex({ [key]: 1 }, {
+            unique: true,
+          });
+        } catch (err: any) {
+          const errorCode = err?.code || err?.codeName || 'unknown';
+          const errorMessage = err?.message || String(err);
+          logger.log(
+            'error',
+            `Failed to create unique index on field "${key}" in collection "${this.collectionName}". ` +
+            `MongoDB error [${errorCode}]: ${errorMessage}. ` +
+            `Uniqueness constraint on "${key}" is NOT enforced.`
+          );
+          if (errorCode === 11000 || errorCode === 'DuplicateKey' || String(errorMessage).includes('E11000')) {
+            await this.logDuplicatesForField(key);
+          }
+        }
       }
     }
   }
@@ -245,16 +266,66 @@ export class SmartdataCollection<T> {
       // Check if we've already created this index
       const indexKey = indexDef.field;
       if (!this.regularIndexes.some(i => i.field === indexKey)) {
-        await this.mongoDbCollection.createIndex(
-          { [indexDef.field]: 1 }, // Simple single-field index
-          indexDef.options
-        );
-        // Track that we've created this index
+        // Claim the slot immediately to prevent concurrent retries
         this.regularIndexes.push(indexDef);
+        try {
+          await this.mongoDbCollection.createIndex(
+            { [indexDef.field]: 1 }, // Simple single-field index
+            indexDef.options
+          );
+        } catch (err: any) {
+          const errorCode = err?.code || err?.codeName || 'unknown';
+          const errorMessage = err?.message || String(err);
+          logger.log(
+            'warn',
+            `Failed to create index on field "${indexKey}" in collection "${this.collectionName}". ` +
+            `MongoDB error [${errorCode}]: ${errorMessage}.`
+          );
+          if (
+            indexDef.options?.unique &&
+            (errorCode === 11000 || errorCode === 'DuplicateKey' || String(errorMessage).includes('E11000'))
+          ) {
+            await this.logDuplicatesForField(indexKey);
+          }
+        }
       }
     }
   }
 
+  /**
+   * Logs duplicate values for a field to help diagnose unique index creation failures.
+   */
+  private async logDuplicatesForField(field: string): Promise<void> {
+    try {
+      const pipeline = [
+        { $group: { _id: `$${field}`, count: { $sum: 1 }, ids: { $push: '$_id' } } },
+        { $match: { count: { $gt: 1 } } },
+        { $limit: 5 },
+      ];
+      const duplicates = await this.mongoDbCollection.aggregate(pipeline).toArray();
+      if (duplicates.length > 0) {
+        for (const dup of duplicates) {
+          logger.log(
+            'warn',
+            `Duplicate values for "${field}" in "${this.collectionName}": ` +
+            `value=${JSON.stringify(dup._id)} appears ${dup.count} times ` +
+            `(document _ids: ${JSON.stringify(dup.ids.slice(0, 5))})`
+          );
+        }
+        logger.log(
+          'warn',
+          `Unique index on "${field}" in "${this.collectionName}" was NOT created. ` +
+          `Resolve duplicates and restart to enforce uniqueness.`
+        );
+      }
+    } catch (aggErr: any) {
+      logger.log(
+        'warn',
+        `Could not identify duplicate documents for field "${field}" in "${this.collectionName}": ${aggErr?.message || String(aggErr)}`
+      );
+    }
+  }
+
   /**
    * adds a validation function that all newly inserted and updated objects have to pass
    */
@@ -295,6 +366,28 @@ export class SmartdataCollection<T> {
     const cursor = this.mongoDbCollection.find(filterObject, { session: opts?.session });
     const result = await cursor.toArray();
     cursor.close();
+
+    // In-memory check for duplicate _id values (should never happen)
+    if (result.length > 0) {
+      const idSet = new Set<string>();
+      const duplicateIds: string[] = [];
+      for (const doc of result) {
+        const idStr = String(doc._id);
+        if (idSet.has(idStr)) {
+          duplicateIds.push(idStr);
+        } else {
+          idSet.add(idStr);
+        }
+      }
+      if (duplicateIds.length > 0) {
+        logger.log(
+          'error',
+          `Integrity issue in "${this.collectionName}": found ${duplicateIds.length} duplicate _id values ` +
+          `in findAll results: [${duplicateIds.slice(0, 5).join(', ')}]. This should never happen.`
+        );
+      }
+    }
+
     return result;
   }
 
@@ -346,11 +439,11 @@ export class SmartdataCollection<T> {
   ): Promise<any> {
     await this.init();
     await this.checkDoc(dbDocArg);
-    this.markUniqueIndexes(dbDocArg.uniqueIndexes);
-    
+    await this.markUniqueIndexes(dbDocArg.uniqueIndexes);
+
     // Create regular indexes if available
     if (dbDocArg.regularIndexes && dbDocArg.regularIndexes.length > 0) {
-      this.createRegularIndexes(dbDocArg.regularIndexes);
+      await this.createRegularIndexes(dbDocArg.regularIndexes);
     }
     
     const saveableObject = await dbDocArg.createSavableObject() as any;
@@ -367,6 +460,7 @@ export class SmartdataCollection<T> {
   ): Promise<any> {
     await this.init();
     await this.checkDoc(dbDocArg);
+    await this.markUniqueIndexes(dbDocArg.uniqueIndexes);
     const identifiableObject = await dbDocArg.createIdentifiableObject();
     const saveableObject = await dbDocArg.createSavableObject() as any;
     const updateableObject: any = {};
@@ -402,6 +496,74 @@ export class SmartdataCollection<T> {
     return this.mongoDbCollection.countDocuments(filterObject, { session: opts?.session });
   }
 
+  /**
+   * Runs an integrity check on the collection.
+   * Compares estimated vs actual document count and checks for duplicates on unique index fields.
+   */
+  public async checkCollectionIntegrity(): Promise<{
+    ok: boolean;
+    estimatedCount: number;
+    actualCount: number;
+    duplicateFields: Array<{ field: string; duplicateValues: number }>;
+  }> {
+    await this.init();
+    const result = {
+      ok: true,
+      estimatedCount: 0,
+      actualCount: 0,
+      duplicateFields: [] as Array<{ field: string; duplicateValues: number }>,
+    };
+
+    try {
+      result.estimatedCount = await this.mongoDbCollection.estimatedDocumentCount();
+      result.actualCount = await this.mongoDbCollection.countDocuments({});
+
+      if (result.estimatedCount !== result.actualCount) {
+        result.ok = false;
+        logger.log(
+          'warn',
+          `Integrity check on "${this.collectionName}": estimatedDocumentCount=${result.estimatedCount} ` +
+          `but countDocuments=${result.actualCount}. Possible data inconsistency.`
+        );
+      }
+
+      // Check for duplicates on each tracked unique index field
+      for (const field of this.uniqueIndexes) {
+        try {
+          const pipeline = [
+            { $group: { _id: `$${field}`, count: { $sum: 1 } } },
+            { $match: { count: { $gt: 1 } } },
+            { $count: 'total' },
+          ];
+          const countResult = await this.mongoDbCollection.aggregate(pipeline).toArray();
+          const dupCount = countResult[0]?.total || 0;
+          if (dupCount > 0) {
+            result.ok = false;
+            result.duplicateFields.push({ field, duplicateValues: dupCount });
+            logger.log(
+              'warn',
+              `Integrity check on "${this.collectionName}": field "${field}" has ${dupCount} values with duplicates ` +
+              `despite being marked as unique.`
+            );
+          }
+        } catch (fieldErr: any) {
+          logger.log(
+            'warn',
+            `Integrity check: could not verify uniqueness of "${field}" in "${this.collectionName}": ${fieldErr?.message || String(fieldErr)}`
+          );
+        }
+      }
+    } catch (err: any) {
+      result.ok = false;
+      logger.log(
+        'error',
+        `Integrity check failed for "${this.collectionName}": ${err?.message || String(err)}`
+      );
+    }
+
+    return result;
+  }
+
   /**
    * checks a Doc for constraints
    * if this.objectValidation is not set it passes.

ts/classes.doc.ts

@@ -597,6 +597,17 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
     return await collection.getCount(filterArg);
   }
 
+  /**
+   * Runs an integrity check on this collection.
+   * Returns a summary with estimated vs actual counts and any duplicate unique fields.
+   */
+  public static async checkCollectionIntegrity<T>(
+    this: plugins.tsclass.typeFest.Class<T>,
+  ) {
+    const collection: SmartdataCollection<T> = (this as any).collection;
+    return await collection.checkCollectionIntegrity();
+  }
+
   /**
    * Create a MongoDB filter from a Lucene query string
    * @param luceneQuery Lucene query string