fix(core): Improve error handling and logging; enhance search query sanitization; update dependency versions and documentation

2025-08-12 11:25:42 +00:00
parent a91fac450a
commit e58c0fd215
17 changed files with 3068 additions and 1596 deletions
--- a/ts/00_commitinfo_data.ts
+++ b/ts/00_commitinfo_data.ts
@@ -3,6 +3,6 @@
 */
 export const commitinfo = {
  name: '@push.rocks/smartdata',
-  version: '5.16.0',
+  version: '5.16.1',
  description: 'An advanced library for NoSQL data organization and manipulation using TypeScript with support for MongoDB, data validation, collections, and custom data types.'
 }
--- a/ts/classes.collection.ts
+++ b/ts/classes.collection.ts
@@ -4,6 +4,7 @@ import { SmartdataDbCursor } from './classes.cursor.js';
 import { SmartDataDbDoc, type IIndexOptions } from './classes.doc.js';
 import { SmartdataDbWatcher } from './classes.watcher.js';
 import { CollectionFactory } from './classes.collectionfactory.js';
+import { logger } from './logging.js';

 export interface IFindOptions {
  limit?: number;
@@ -161,7 +162,7 @@ export class SmartdataCollection<T> {
      });
      if (!wantedCollection) {
        await this.smartdataDb.mongoDb.createCollection(this.collectionName);
-        console.log(`Successfully initiated Collection ${this.collectionName}`);
+        logger.log('info', `Successfully initiated Collection ${this.collectionName}`);
      }
      this.mongoDbCollection = this.smartdataDb.mongoDb.collection(this.collectionName);
      // Auto-create a compound text index on all searchable fields
@@ -182,10 +183,10 @@ export class SmartdataCollection<T> {
  /**
   * mark unique index
   */
-  public markUniqueIndexes(keyArrayArg: string[] = []) {
+  public async markUniqueIndexes(keyArrayArg: string[] = []) {
    for (const key of keyArrayArg) {
      if (!this.uniqueIndexes.includes(key)) {
-        this.mongoDbCollection.createIndex(key, {
+        await this.mongoDbCollection.createIndex({ [key]: 1 }, {
          unique: true,
        });
        // make sure we only call this once and not for every doc we create
@@ -197,12 +198,12 @@ export class SmartdataCollection<T> {
  /**
   * creates regular indexes for the collection
   */
-  public createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
+  public async createRegularIndexes(indexesArg: Array<{field: string, options: IIndexOptions}> = []) {
    for (const indexDef of indexesArg) {
      // Check if we've already created this index
      const indexKey = indexDef.field;
      if (!this.regularIndexes.some(i => i.field === indexKey)) {
-        this.mongoDbCollection.createIndex(
+        await this.mongoDbCollection.createIndex(
          { [indexDef.field]: 1 }, // Simple single-field index
          indexDef.options
        );
@@ -275,7 +276,7 @@ export class SmartdataCollection<T> {
      fullDocument:
        fullDocument === undefined
          ? 'updateLookup'
-          : fullDocument === true
+          : (fullDocument as any) === true
          ? 'updateLookup'
          : fullDocument,
    } as any;
--- a/ts/classes.db.ts
+++ b/ts/classes.db.ts
@@ -35,24 +35,43 @@ export class SmartdataDb {
   * connects to the database that was specified during instance creation
   */
  public async init(): Promise<any> {
-    const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
-      .replace('<USERNAME>', this.smartdataOptions.mongoDbUser)
-      .replace('<username>', this.smartdataOptions.mongoDbUser)
-      .replace('<USER>', this.smartdataOptions.mongoDbUser)
-      .replace('<user>', this.smartdataOptions.mongoDbUser)
-      .replace('<PASSWORD>', this.smartdataOptions.mongoDbPass)
-      .replace('<password>', this.smartdataOptions.mongoDbPass)
-      .replace('<DBNAME>', this.smartdataOptions.mongoDbName)
-      .replace('<dbname>', this.smartdataOptions.mongoDbName);
+    try {
+      // Safely encode credentials to handle special characters
+      const encodedUser = this.smartdataOptions.mongoDbUser 
+        ? encodeURIComponent(this.smartdataOptions.mongoDbUser) 
+        : '';
+      const encodedPass = this.smartdataOptions.mongoDbPass 
+        ? encodeURIComponent(this.smartdataOptions.mongoDbPass) 
+        : '';
+      
+      const finalConnectionUrl = this.smartdataOptions.mongoDbUrl
+        .replace('<USERNAME>', encodedUser)
+        .replace('<username>', encodedUser)
+        .replace('<USER>', encodedUser)
+        .replace('<user>', encodedUser)
+        .replace('<PASSWORD>', encodedPass)
+        .replace('<password>', encodedPass)
+        .replace('<DBNAME>', this.smartdataOptions.mongoDbName)
+        .replace('<dbname>', this.smartdataOptions.mongoDbName);

-    this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, {
-      maxPoolSize: 100,
-      maxIdleTimeMS: 10,
-    });
-    this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
-    this.status = 'connected';
-    this.statusConnectedDeferred.resolve();
-    console.log(`Connected to database ${this.smartdataOptions.mongoDbName}`);
+      const clientOptions: plugins.mongodb.MongoClientOptions = {
+        maxPoolSize: (this.smartdataOptions as any).maxPoolSize ?? 100,
+        maxIdleTimeMS: (this.smartdataOptions as any).maxIdleTimeMS ?? 300000, // 5 minutes default
+        serverSelectionTimeoutMS: (this.smartdataOptions as any).serverSelectionTimeoutMS ?? 30000,
+        retryWrites: true,
+      };
+
+      this.mongoDbClient = await plugins.mongodb.MongoClient.connect(finalConnectionUrl, clientOptions);
+      this.mongoDb = this.mongoDbClient.db(this.smartdataOptions.mongoDbName);
+      this.status = 'connected';
+      this.statusConnectedDeferred.resolve();
+      logger.log('info', `Connected to database ${this.smartdataOptions.mongoDbName}`);
+    } catch (error) {
+      this.status = 'disconnected';
+      this.statusConnectedDeferred.reject(error);
+      logger.log('error', `Failed to connect to database ${this.smartdataOptions.mongoDbName}: ${error.message}`);
+      throw error;
+    }
  }

  /**
--- a/ts/classes.distributedcoordinator.ts
+++ b/ts/classes.distributedcoordinator.ts
@@ -3,6 +3,7 @@ import { SmartdataDb } from './classes.db.js';
 import { managed, setDefaultManagerForDoc } from './classes.collection.js';
 import { SmartDataDbDoc, svDb, unI } from './classes.doc.js';
 import { SmartdataDbWatcher } from './classes.watcher.js';
+import { logger } from './logging.js';

@managed()
 export class DistributedClass extends SmartDataDbDoc<DistributedClass, DistributedClass> {
@@ -63,11 +64,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
        this.ownInstance.data.elected = false;
      }
      if (this.ownInstance?.data.status === 'stopped') {
-        console.log(`stopping a distributed instance that has not been started yet.`);
+        logger.log('warn', `stopping a distributed instance that has not been started yet.`);
      }
      this.ownInstance.data.status = 'stopped';
      await this.ownInstance.save();
-      console.log(`stopped ${this.ownInstance.id}`);
+      logger.log('info', `stopped ${this.ownInstance.id}`);
    });
  }

@@ -83,17 +84,17 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
  public async sendHeartbeat() {
    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
      if (this.ownInstance.data.status === 'stopped') {
-        console.log(`aborted sending heartbeat because status is stopped`);
+        logger.log('debug', `aborted sending heartbeat because status is stopped`);
        return;
      }
      await this.ownInstance.updateFromDb();
      this.ownInstance.data.lastUpdated = Date.now();
      await this.ownInstance.save();
-      console.log(`sent heartbeat for ${this.ownInstance.id}`);
+      logger.log('debug', `sent heartbeat for ${this.ownInstance.id}`);
      const allInstances = DistributedClass.getInstances({});
    });
    if (this.ownInstance.data.status === 'stopped') {
-      console.log(`aborted sending heartbeat because status is stopped`);
+      logger.log('info', `aborted sending heartbeat because status is stopped`);
      return;
    }
    const eligibleLeader = await this.getEligibleLeader();
@@ -120,7 +121,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
        await this.ownInstance.save();
      });
    } else {
-      console.warn(`distributed instance already initialized`);
+      logger.log('warn', `distributed instance already initialized`);
    }

    // lets enable the heartbeat
@@ -149,24 +150,24 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
  public async checkAndMaybeLead() {
    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
      this.ownInstance.data.status = 'initializing';
-      this.ownInstance.save();
+      await this.ownInstance.save();
    });
    if (await this.getEligibleLeader()) {
      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
        await this.ownInstance.updateFromDb();
        this.ownInstance.data.status = 'settled';
        await this.ownInstance.save();
-        console.log(`${this.ownInstance.id} settled as follower`);
+        logger.log('info', `${this.ownInstance.id} settled as follower`);
      });
      return;
    } else if (
      (await DistributedClass.getInstances({})).find((instanceArg) => {
-        instanceArg.data.status === 'bidding' &&
+        return instanceArg.data.status === 'bidding' &&
          instanceArg.data.biddingStartTime <= Date.now() - 4000 &&
          instanceArg.data.biddingStartTime >= Date.now() - 30000;
      })
    ) {
-      console.log('too late to the bidding party... waiting for next round.');
+      logger.log('info', 'too late to the bidding party... waiting for next round.');
      return;
    } else {
      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
@@ -175,9 +176,9 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
        this.ownInstance.data.biddingStartTime = Date.now();
        this.ownInstance.data.biddingShortcode = plugins.smartunique.shortId();
        await this.ownInstance.save();
-        console.log('bidding code stored.');
+        logger.log('info', 'bidding code stored.');
      });
-      console.log(`bidding for leadership...`);
+      logger.log('info', `bidding for leadership...`);
      await plugins.smartdelay.delayFor(plugins.smarttime.getMilliSecondsFromUnits({ seconds: 5 }));
      await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
        let biddingInstances = await DistributedClass.getInstances({});
@@ -187,7 +188,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
            instanceArg.data.lastUpdated >=
              Date.now() - plugins.smarttime.getMilliSecondsFromUnits({ seconds: 10 }),
        );
-        console.log(`found ${biddingInstances.length} bidding instances...`);
+        logger.log('info', `found ${biddingInstances.length} bidding instances...`);
        this.ownInstance.data.elected = true;
        for (const biddingInstance of biddingInstances) {
          if (biddingInstance.data.biddingShortcode < this.ownInstance.data.biddingShortcode) {
@@ -195,7 +196,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
          }
        }
        await plugins.smartdelay.delayFor(5000);
-        console.log(`settling with status elected = ${this.ownInstance.data.elected}`);
+        logger.log('info', `settling with status elected = ${this.ownInstance.data.elected}`);
        this.ownInstance.data.status = 'settled';
        await this.ownInstance.save();
      });
@@ -226,11 +227,11 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
    this.distributedWatcher.changeSubject.subscribe({
      next: async (distributedDoc) => {
        if (!distributedDoc) {
-          console.log(`registered deletion of instance...`);
+          logger.log('info', `registered deletion of instance...`);
          return;
        }
-        console.log(distributedDoc);
-        console.log(`registered change for ${distributedDoc.id}`);
+        logger.log('info', distributedDoc);
+        logger.log('info', `registered change for ${distributedDoc.id}`);
        distributedDoc;
      },
    });
@@ -252,7 +253,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
  ): Promise<plugins.taskbuffer.distributedCoordination.IDistributedTaskRequestResult> {
    await this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
      if (!this.ownInstance) {
-        console.error('instance need to be started first...');
+        logger.log('error', 'instance need to be started first...');
        return;
      }
      await this.ownInstance.updateFromDb();
@@ -268,7 +269,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
      return taskRequestResult;
    });
    if (!result) {
-      console.warn('no result found for task request...');
+      logger.log('warn', 'no result found for task request...');
      return null;
    }
    return result;
@@ -285,7 +286,7 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
        );
      });
      if (!existingInfoBasis) {
-        console.warn('trying to update a non existing task request... aborting!');
+        logger.log('warn', 'trying to update a non existing task request... aborting!');
        return;
      }
      Object.assign(existingInfoBasis, infoBasisArg);
@@ -293,8 +294,10 @@ export class SmartdataDistributedCoordinator extends plugins.taskbuffer.distribu
      plugins.smartdelay.delayFor(60000).then(() => {
        this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
          const indexToRemove = this.ownInstance.data.taskRequests.indexOf(existingInfoBasis);
-          this.ownInstance.data.taskRequests.splice(indexToRemove, indexToRemove);
-          await this.ownInstance.save();
+          if (indexToRemove >= 0) {
+            this.ownInstance.data.taskRequests.splice(indexToRemove, 1);
+            await this.ownInstance.save();
+          }
        });
      });
    });
--- a/ts/classes.doc.ts
+++ b/ts/classes.doc.ts
@@ -1,6 +1,7 @@
 import * as plugins from './plugins.js';

 import { SmartdataDb } from './classes.db.js';
+import { logger } from './logging.js';
 import { SmartdataDbCursor } from './classes.cursor.js';
 import { type IManager, SmartdataCollection } from './classes.collection.js';
 import { SmartdataDbWatcher } from './classes.watcher.js';
@@ -31,7 +32,7 @@ export type TDocCreation = 'db' | 'new' | 'mixed';

 export function globalSvDb() {
  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
-    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    logger.log('debug', `called svDb() on >${target.constructor.name}.${key}<`);
    if (!target.globalSaveableProperties) {
      target.globalSaveableProperties = [];
    }
@@ -54,7 +55,7 @@ export interface SvDbOptions {
 */
 export function svDb(options?: SvDbOptions) {
  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
-    console.log(`called svDb() on >${target.constructor.name}.${key}<`);
+    logger.log('debug', `called svDb() on >${target.constructor.name}.${key}<`);
    if (!target.saveableProperties) {
      target.saveableProperties = [];
    }
@@ -94,7 +95,7 @@ function escapeForRegex(input: string): string {
 */
 export function unI() {
  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
-    console.log(`called unI on >>${target.constructor.name}.${key}<<`);
+    logger.log('debug', `called unI on >>${target.constructor.name}.${key}<<`);

    // mark the index as unique
    if (!target.uniqueIndexes) {
@@ -126,7 +127,7 @@ export interface IIndexOptions {
 */
 export function index(options?: IIndexOptions) {
  return (target: SmartDataDbDoc<unknown, unknown>, key: string) => {
-    console.log(`called index() on >${target.constructor.name}.${key}<`);
+    logger.log('debug', `called index() on >${target.constructor.name}.${key}<`);
    
    // Initialize regular indexes array if it doesn't exist
    if (!target.regularIndexes) {
@@ -152,7 +153,8 @@ export function index(options?: IIndexOptions) {

 export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {
  // Special case: detect MongoDB operators and pass them through directly
-  const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$where', '$regex'];
+  // SECURITY: Removed $where to prevent server-side JS execution
+  const topLevelOperators = ['$and', '$or', '$nor', '$not', '$text', '$regex'];
  for (const key of Object.keys(filterArg)) {
    if (topLevelOperators.includes(key)) {
      return filterArg; // Return the filter as-is for MongoDB operators
@@ -164,11 +166,16 @@ export const convertFilterForMongoDb = (filterArg: { [key: string]: any }) => {

  const convertFilterArgument = (keyPathArg2: string, filterArg2: any) => {
    if (Array.isArray(filterArg2)) {
-      // Directly assign arrays (they might be using operators like $in or $all)
-      convertFilterArgument(keyPathArg2, filterArg2[0]);
+      // FIX: Properly handle arrays for operators like $in, $all, or plain equality
+      convertedFilter[keyPathArg2] = filterArg2;
+      return;
    } else if (typeof filterArg2 === 'object' && filterArg2 !== null) {
      for (const key of Object.keys(filterArg2)) {
        if (key.startsWith('$')) {
+          // Prevent dangerous operators
+          if (key === '$where') {
+            throw new Error('$where operator is not allowed for security reasons');
+          }
          convertedFilter[keyPathArg2] = filterArg2;
          return;
        } else if (key.includes('.')) {
@@ -614,7 +621,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
        this.creationStatus = 'db';
        break;
      default:
-        console.error('neither new nor in db?');
+        logger.log('error', 'neither new nor in db?');
    }
    // allow hook after saving
    if (typeof (this as any).afterSave === 'function') {
@@ -661,8 +668,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
  /**
   * updates an object from db
   */
-  public async updateFromDb() {
+  public async updateFromDb(): Promise<boolean> {
    const mongoDbNativeDoc = await this.collection.findOne(await this.createIdentifiableObject());
+    if (!mongoDbNativeDoc) {
+      return false; // Document not found in database
+    }
    for (const key of Object.keys(mongoDbNativeDoc)) {
      const rawValue = mongoDbNativeDoc[key];
      const optionsMap = (this.constructor as any)._svDbOptions || {};
@@ -671,6 +681,7 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
        ? opts.deserialize(rawValue)
        : rawValue;
    }
+    return true;
  }

  /**
@@ -678,7 +689,9 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   */
  public async createSavableObject(): Promise<TImplements> {
    const saveableObject: unknown = {}; // is not exposed to outside, so any is ok here
-    const saveableProperties = [...this.globalSaveableProperties, ...this.saveableProperties];
+    const globalProps = this.globalSaveableProperties || [];
+    const specificProps = this.saveableProperties || [];
+    const saveableProperties = [...globalProps, ...specificProps];
    // apply custom serialization if configured
    const optionsMap = (this.constructor as any)._svDbOptions || {};
    for (const propertyNameString of saveableProperties) {
--- a/ts/classes.easystore.ts
+++ b/ts/classes.easystore.ts
@@ -18,7 +18,7 @@ export class EasyStore<T> {
      public nameId: string;

      @svDb()
-      public ephermal: {
+      public ephemeral: {
        activated: boolean;
        timeout: number;
      };
@@ -32,8 +32,8 @@ export class EasyStore<T> {
    return SmartdataEasyStore;
  })();

-  constructor(nameIdArg: string, smnartdataDbRefArg: SmartdataDb) {
-    this.smartdataDbRef = smnartdataDbRefArg;
+  constructor(nameIdArg: string, smartdataDbRefArg: SmartdataDb) {
+    this.smartdataDbRef = smartdataDbRefArg;
    this.nameId = nameIdArg;
  }

@@ -110,10 +110,12 @@ export class EasyStore<T> {
    await easyStore.save();
  }

-  public async cleanUpEphermal() {
-    while (
-      (await this.smartdataDbRef.statusConnectedDeferred.promise) &&
-      this.smartdataDbRef.status === 'connected'
-    ) {}
+  public async cleanUpEphemeral() {
+    // Clean up ephemeral data periodically while connected
+    while (this.smartdataDbRef.status === 'connected') {
+      await plugins.smartdelay.delayFor(60000); // Check every minute
+      // TODO: Implement actual cleanup logic for ephemeral data
+      // For now, this prevents the infinite CPU loop
+    }
  }
 }
--- a/ts/classes.lucene.adapter.ts
+++ b/ts/classes.lucene.adapter.ts
@@ -2,6 +2,7 @@
 * Lucene to MongoDB query adapter for SmartData
 */
 import * as plugins from './plugins.js';
+import { logger } from './logging.js';

 // Types
 type NodeType =
@@ -754,7 +755,7 @@ export class SmartdataLuceneAdapter {
      // Transform the AST to a MongoDB query
      return this.transformWithFields(ast, fieldsToSearch);
    } catch (error) {
-      console.error(`Failed to convert Lucene query "${luceneQuery}":`, error);
+      logger.log('error', `Failed to convert Lucene query "${luceneQuery}":`, error);
      throw new Error(`Failed to convert Lucene query: ${error}`);
    }
  }