smartdata/test/test.filters.ts

import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as smartmongo from '@push.rocks/smartmongo';
import * as smartunique from '@push.rocks/smartunique';
import * as smartdata from '../ts/index.js';

const { SmartdataDb, Collection, svDb, unI, index } = smartdata;

let smartmongoInstance: smartmongo.SmartMongo;
let testDb: smartdata.SmartdataDb;

// Define test document classes
@Collection(() => testDb)
class TestUser extends smartdata.SmartDataDbDoc<TestUser, TestUser> {
  @unI()
  public id: string = smartunique.shortId();

  @svDb()
  public name: string;

  @svDb()
  public age: number;

  @svDb()
  public email: string;

  @svDb()
  public roles: string[];

  @svDb()
  public tags: string[];

  @svDb()
  public status: 'active' | 'inactive' | 'pending';

  @svDb()
  public metadata: {
    lastLogin?: Date;
    loginCount?: number;
    preferences?: Record<string, any>;
  };

  @svDb()
  public scores: number[];

  constructor(data: Partial<TestUser> = {}) {
    super();
    Object.assign(this, data);
  }
}

@Collection(() => testDb)
class TestOrder extends smartdata.SmartDataDbDoc<TestOrder, TestOrder> {
  @unI()
  public id: string = smartunique.shortId();

  @svDb()
  public userId: string;

  @svDb()
  public items: Array<{
    product: string;
    quantity: number;
    price: number;
  }>;

  @svDb()
  public totalAmount: number;

  @svDb()
  public status: string;

  @svDb()
  public tags: string[];

  constructor(data: Partial<TestOrder> = {}) {
    super();
    Object.assign(this, data);
  }
}

// Setup and teardown
tap.test('should create a test database instance', async () => {
  smartmongoInstance = await smartmongo.SmartMongo.createAndStart();
  testDb = new smartdata.SmartdataDb(await smartmongoInstance.getMongoDescriptor());
  await testDb.init();
  expect(testDb).toBeInstanceOf(SmartdataDb);
});

tap.test('should create test data', async () => {
  // Create test users
  const users = [
    new TestUser({
      name: 'John Doe',
      age: 30,
      email: 'john@example.com',
      roles: ['admin', 'user'],
      tags: ['javascript', 'nodejs', 'mongodb'],
      status: 'active',
      metadata: { loginCount: 5, lastLogin: new Date() },
      scores: [85, 90, 78]
    }),
    new TestUser({
      name: 'Jane Smith',
      age: 25,
      email: 'jane@example.com',
      roles: ['user'],
      tags: ['python', 'mongodb'],
      status: 'active',
      metadata: { loginCount: 3 },
      scores: [92, 88, 95]
    }),
    new TestUser({
      name: 'Bob Johnson',
      age: 35,
      email: 'bob@example.com',
      roles: ['moderator', 'user'],
      tags: ['javascript', 'react', 'nodejs'],
      status: 'inactive',
      metadata: { loginCount: 0 },
      scores: [70, 75, 80]
    }),
    new TestUser({
      name: 'Alice Brown',
      age: 28,
      email: 'alice@example.com',
      roles: ['admin'],
      tags: ['typescript', 'angular', 'mongodb'],
      status: 'active',
      metadata: { loginCount: 10 },
      scores: [95, 98, 100]
    }),
    new TestUser({
      name: 'Charlie Wilson',
      age: 22,
      email: 'charlie@example.com',
      roles: ['user'],
      tags: ['golang', 'kubernetes'],
      status: 'pending',
      metadata: { loginCount: 1 },
      scores: [60, 65]
    })
  ];

  for (const user of users) {
    await user.save();
  }

  // Create test orders
  const orders = [
    new TestOrder({
      userId: users[0].id,
      items: [
        { product: 'laptop', quantity: 1, price: 1200 },
        { product: 'mouse', quantity: 2, price: 25 }
      ],
      totalAmount: 1250,
      status: 'completed',
      tags: ['electronics', 'priority']
    }),
    new TestOrder({
      userId: users[1].id,
      items: [
        { product: 'book', quantity: 3, price: 15 },
        { product: 'pen', quantity: 5, price: 2 }
      ],
      totalAmount: 55,
      status: 'pending',
      tags: ['stationery']
    }),
    new TestOrder({
      userId: users[0].id,
      items: [
        { product: 'laptop', quantity: 2, price: 1200 },
        { product: 'keyboard', quantity: 2, price: 80 }
      ],
      totalAmount: 2560,
      status: 'processing',
      tags: ['electronics', 'bulk']
    })
  ];

  for (const order of orders) {
    await order.save();
  }

  const savedUsers = await TestUser.getInstances({});
  const savedOrders = await TestOrder.getInstances({});
  expect(savedUsers.length).toEqual(5);
  expect(savedOrders.length).toEqual(3);
});

// ============= BASIC FILTER TESTS =============
tap.test('should filter by simple equality', async () => {
  const users = await TestUser.getInstances({ name: 'John Doe' });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should filter by multiple fields (implicit AND)', async () => {
  const users = await TestUser.getInstances({ 
    status: 'active',
    age: 30
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should filter by nested object fields', async () => {
  const users = await TestUser.getInstances({
    'metadata.loginCount': 5
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

// ============= COMPREHENSIVE NESTED FILTER TESTS =============
tap.test('should filter by nested object with direct object syntax', async () => {
  // Direct nested object matching (exact match)
  const users = await TestUser.getInstances({
    metadata: { 
      loginCount: 5,
      lastLogin: (await TestUser.getInstances({}))[0].metadata.lastLogin // Get the exact date
    }
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should filter by partial nested object match', async () => {
  // When using object syntax, only specified fields must match
  const users = await TestUser.getInstances({
    metadata: { loginCount: 5 }  // Only checks loginCount, ignores other fields
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should combine nested object and dot notation', async () => {
  const users = await TestUser.getInstances({
    metadata: { loginCount: { $gte: 3 } },  // Object syntax with operator
    'metadata.loginCount': { $lte: 10 }     // Dot notation with operator
  });
  expect(users.length).toEqual(3);  // Jane (3), John (5), and Alice (10) have loginCount between 3-10
});

tap.test('should filter nested fields with operators using dot notation', async () => {
  const users = await TestUser.getInstances({
    'metadata.loginCount': { $gte: 5 }
  });
  expect(users.length).toEqual(2);  // John (5) and Alice (10)
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'John Doe']);
});

tap.test('should filter nested fields with multiple operators', async () => {
  const users = await TestUser.getInstances({
    'metadata.loginCount': { $gte: 3, $lt: 10 }
  });
  expect(users.length).toEqual(2);  // Jane (3) and John (5)
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Jane Smith', 'John Doe']);
});

tap.test('should handle deeply nested object structures', async () => {
  // First, create a user with deep nesting in preferences
  const deepUser = new TestUser({
    name: 'Deep Nester',
    age: 40,
    email: 'deep@example.com',
    roles: ['admin'],
    tags: [],
    status: 'active',
    metadata: {
      loginCount: 1,
      preferences: {
        theme: {
          colors: {
            primary: '#000000',
            secondary: '#ffffff'
          },
          fonts: {
            heading: 'Arial',
            body: 'Helvetica'
          }
        },
        notifications: {
          email: true,
          push: false
        }
      }
    },
    scores: []
  });
  await deepUser.save();

  // Test deep nesting with dot notation
  const deepResults = await TestUser.getInstances({
    'metadata.preferences.theme.colors.primary': '#000000'
  });
  expect(deepResults.length).toEqual(1);
  expect(deepResults[0].name).toEqual('Deep Nester');

  // Test deep nesting with operators
  const boolResults = await TestUser.getInstances({
    'metadata.preferences.notifications.email': { $eq: true }
  });
  expect(boolResults.length).toEqual(1);
  expect(boolResults[0].name).toEqual('Deep Nester');

  // Clean up
  await deepUser.delete();
});

tap.test('should filter arrays of nested objects using $elemMatch', async () => {
  const orders = await TestOrder.getInstances({
    items: {
      $elemMatch: {
        product: 'laptop',
        price: { $gte: 1000 }
      }
    }
  });
  expect(orders.length).toEqual(2);  // Both laptop orders have price >= 1000
});

tap.test('should filter nested arrays with dot notation', async () => {
  // Query for any order that has an item with specific product
  const orders = await TestOrder.getInstances({
    'items.product': 'laptop'
  });
  expect(orders.length).toEqual(2);  // Two orders contain laptops
});

tap.test('should combine nested object filters with logical operators', async () => {
  const users = await TestUser.getInstances({
    $or: [
      { 'metadata.loginCount': { $gte: 10 } },  // Alice has 10
      { 
        $and: [
          { 'metadata.loginCount': { $lt: 5 } },  // Jane has 3, Bob has 0, Charlie has 1
          { status: 'active' }  // Jane is active, Bob is inactive, Charlie is pending
        ]
      }
    ]
  });
  expect(users.length).toEqual(2);  // Alice (loginCount >= 10), Jane (loginCount < 5 AND active)
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'Jane Smith']);
});

tap.test('should handle null and undefined in nested fields', async () => {
  // Users without lastLogin
  const noLastLogin = await TestUser.getInstances({
    'metadata.lastLogin': { $exists: false }
  });
  expect(noLastLogin.length).toEqual(4);  // Everyone except John

  // Users with preferences (none have it set)
  const withPreferences = await TestUser.getInstances({
    'metadata.preferences': { $exists: true }
  });
  expect(withPreferences.length).toEqual(0);
});

tap.test('should filter nested arrays by size', async () => {
  // Create an order with specific number of items
  const multiItemOrder = new TestOrder({
    userId: 'test-user',
    items: [
      { product: 'item1', quantity: 1, price: 10 },
      { product: 'item2', quantity: 2, price: 20 },
      { product: 'item3', quantity: 3, price: 30 },
      { product: 'item4', quantity: 4, price: 40 }
    ],
    totalAmount: 100,
    status: 'pending',
    tags: ['test']
  });
  await multiItemOrder.save();

  const fourItemOrders = await TestOrder.getInstances({
    items: { $size: 4 }
  });
  expect(fourItemOrders.length).toEqual(1);

  // Clean up
  await multiItemOrder.delete();
});

tap.test('should handle nested field comparison between documents', async () => {
  // Find users where loginCount equals their age divided by 6 (John: 30/6=5)
  const users = await TestUser.getInstances({
    $and: [
      { 'metadata.loginCount': 5 },
      { age: 30 }
    ]
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should filter using $in on nested fields', async () => {
  const users = await TestUser.getInstances({
    'metadata.loginCount': { $in: [0, 1, 5] }
  });
  expect(users.length).toEqual(3);  // Bob (0), Charlie (1), John (5)
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Bob Johnson', 'Charlie Wilson', 'John Doe']);
});

tap.test('should filter nested arrays with $all', async () => {
  // Create an order with multiple tags
  const taggedOrder = new TestOrder({
    userId: 'test-user',
    items: [{ product: 'test', quantity: 1, price: 10 }],
    totalAmount: 10,
    status: 'completed',
    tags: ['urgent', 'priority', 'electronics']
  });
  await taggedOrder.save();

  const priorityElectronics = await TestOrder.getInstances({
    tags: { $all: ['priority', 'electronics'] }
  });
  expect(priorityElectronics.length).toEqual(2);  // Original order and new one

  // Clean up
  await taggedOrder.delete();
});

// ============= COMPARISON OPERATOR TESTS =============
tap.test('should filter using $gt operator', async () => {
  const users = await TestUser.getInstances({
    age: { $gt: 30 }
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('Bob Johnson');
});

tap.test('should filter using $gte operator', async () => {
  const users = await TestUser.getInstances({
    age: { $gte: 30 }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Bob Johnson', 'John Doe']);
});

tap.test('should filter using $lt operator', async () => {
  const users = await TestUser.getInstances({
    age: { $lt: 25 }
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('Charlie Wilson');
});

tap.test('should filter using $lte operator', async () => {
  const users = await TestUser.getInstances({
    age: { $lte: 25 }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
});

tap.test('should filter using $ne operator', async () => {
  const users = await TestUser.getInstances({
    status: { $ne: 'active' }
  });
  expect(users.length).toEqual(2);
  const statuses = users.map(u => u.status).sort();
  expect(statuses).toEqual(['inactive', 'pending']);
});

tap.test('should filter using multiple comparison operators', async () => {
  const users = await TestUser.getInstances({
    age: { $gte: 25, $lt: 30 }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'Jane Smith']);
});

// ============= ARRAY OPERATOR TESTS =============
tap.test('should filter using $in operator', async () => {
  const users = await TestUser.getInstances({
    status: { $in: ['active', 'pending'] }
  });
  expect(users.length).toEqual(4);
  expect(users.every(u => ['active', 'pending'].includes(u.status))).toEqual(true);
});

tap.test('should filter arrays using $in operator', async () => {
  const users = await TestUser.getInstances({
    roles: { $in: ['admin'] }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'John Doe']);
});

tap.test('should filter using $nin operator', async () => {
  const users = await TestUser.getInstances({
    status: { $nin: ['inactive', 'pending'] }
  });
  expect(users.length).toEqual(3);
  expect(users.every(u => u.status === 'active')).toEqual(true);
});

tap.test('should filter arrays using $all operator', async () => {
  const users = await TestUser.getInstances({
    tags: { $all: ['javascript', 'nodejs'] }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Bob Johnson', 'John Doe']);
});

tap.test('should filter arrays using $size operator', async () => {
  const users = await TestUser.getInstances({
    scores: { $size: 2 }
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('Charlie Wilson');
});

tap.test('should filter arrays using $elemMatch operator', async () => {
  const orders = await TestOrder.getInstances({
    items: {
      $elemMatch: {
        product: 'laptop',
        quantity: { $gte: 2 }
      }
    }
  });
  expect(orders.length).toEqual(1);
  expect(orders[0].totalAmount).toEqual(2560);
});

tap.test('should filter using $elemMatch with single condition', async () => {
  const orders = await TestOrder.getInstances({
    items: {
      $elemMatch: {
        price: { $gt: 100 }
      }
    }
  });
  expect(orders.length).toEqual(2);
  expect(orders.every(o => o.items.some(i => i.price > 100))).toEqual(true);
});

// ============= LOGICAL OPERATOR TESTS =============
tap.test('should filter using $or operator', async () => {
  const users = await TestUser.getInstances({
    $or: [
      { age: { $lt: 25 } },
      { status: 'inactive' }
    ]
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Bob Johnson', 'Charlie Wilson']);
});

tap.test('should filter using $and operator', async () => {
  const users = await TestUser.getInstances({
    $and: [
      { status: 'active' },
      { age: { $gte: 28 } }
    ]
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'John Doe']);
});

tap.test('should filter using $nor operator', async () => {
  const users = await TestUser.getInstances({
    $nor: [
      { status: 'inactive' },
      { age: { $lt: 25 } }
    ]
  });
  expect(users.length).toEqual(3);
  expect(users.every(u => u.status !== 'inactive' && u.age >= 25)).toEqual(true);
});

tap.test('should filter using nested logical operators', async () => {
  const users = await TestUser.getInstances({
    $or: [
      {
        $and: [
          { status: 'active' },
          { roles: { $in: ['admin'] } }
        ]
      },
      { age: { $lt: 23 } }
    ]
  });
  expect(users.length).toEqual(3);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'Charlie Wilson', 'John Doe']);
});

// ============= ELEMENT OPERATOR TESTS =============
tap.test('should filter using $exists operator', async () => {
  const users = await TestUser.getInstances({
    'metadata.lastLogin': { $exists: true }
  });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('John Doe');
});

tap.test('should filter using $exists false', async () => {
  const users = await TestUser.getInstances({
    'metadata.preferences': { $exists: false }
  });
  expect(users.length).toEqual(5);
});

// ============= COMPLEX FILTER TESTS =============
tap.test('should handle complex nested filters', async () => {
  const users = await TestUser.getInstances({
    $and: [
      { status: 'active' },
      {
        $or: [
          { age: { $gte: 30 } },
          { roles: { $all: ['admin'] } }
        ]
      },
      { tags: { $in: ['mongodb'] } }
    ]
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Alice Brown', 'John Doe']);
});

tap.test('should combine multiple operator types', async () => {
  const orders = await TestOrder.getInstances({
    $and: [
      { totalAmount: { $gte: 100 } },
      { status: { $in: ['completed', 'processing'] } },
      { tags: { $in: ['electronics'] } }
    ]
  });
  expect(orders.length).toEqual(2);
  expect(orders.every(o => o.totalAmount >= 100)).toEqual(true);
});

// ============= ERROR HANDLING TESTS =============
tap.test('should throw error for $where operator', async () => {
  let error: Error | null = null;
  try {
    await TestUser.getInstances({
      $where: 'this.age > 25'
    });
  } catch (e) {
    error = e as Error;
  }
  expect(error).toBeTruthy();
  expect(error?.message).toMatch(/\$where.*not allowed/);
});

tap.test('should throw error for invalid $in value', async () => {
  let error: Error | null = null;
  try {
    await TestUser.getInstances({
      status: { $in: 'active' as any } // Should be an array
    });
  } catch (e) {
    error = e as Error;
  }
  expect(error).toBeTruthy();
  expect(error?.message).toMatch(/\$in.*requires.*array/);
});

tap.test('should throw error for invalid $size value', async () => {
  let error: Error | null = null;
  try {
    await TestUser.getInstances({
      scores: { $size: '3' as any } // Should be a number
    });
  } catch (e) {
    error = e as Error;
  }
  expect(error).toBeTruthy();
  expect(error?.message).toMatch(/\$size.*requires.*numeric/);
});

tap.test('should throw error for dots in field names', async () => {
  let error: Error | null = null;
  try {
    await TestUser.getInstances({
      'some.nested.field': { 'invalid.key': 'value' }
    });
  } catch (e) {
    error = e as Error;
  }
  expect(error).toBeTruthy();
  expect(error?.message).toMatch(/keys cannot contain dots/);
});

// ============= EDGE CASE TESTS =============
tap.test('should handle empty filter (return all)', async () => {
  const users = await TestUser.getInstances({});
  expect(users.length).toEqual(5);
});

tap.test('should handle null values in filter', async () => {
  // First, create a user with null email
  const nullUser = new TestUser({
    name: 'Null User',
    age: 40,
    email: null as any,
    roles: ['user'],
    tags: [],
    status: 'active',
    metadata: {},
    scores: []
  });
  await nullUser.save();

  const users = await TestUser.getInstances({ email: null });
  expect(users.length).toEqual(1);
  expect(users[0].name).toEqual('Null User');

  // Clean up
  await nullUser.delete();
});

tap.test('should handle arrays as direct equality match', async () => {
  // This tests that arrays without operators are treated as equality matches
  const users = await TestUser.getInstances({
    roles: ['user'] // Exact match for array
  });
  expect(users.length).toEqual(2);  // Both Jane and Charlie have exactly ['user']
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Charlie Wilson', 'Jane Smith']);
});

tap.test('should handle regex operator', async () => {
  const users = await TestUser.getInstances({
    name: { $regex: '^J', $options: 'i' }
  });
  expect(users.length).toEqual(2);
  const names = users.map(u => u.name).sort();
  expect(names).toEqual(['Jane Smith', 'John Doe']);
});

tap.test('should handle unknown operators by letting MongoDB reject them', async () => {
  // Unknown operators should be passed through to MongoDB, which will reject them
  let error: Error | null = null;
  
  try {
    await TestUser.getInstances({
      age: { $unknownOp: 30 } as any
    });
  } catch (e) {
    error = e as Error;
  }

  expect(error).toBeTruthy();
  expect(error?.message).toMatch(/unknown operator.*\$unknownOp/);
});

// ============= PERFORMANCE TESTS =============
tap.test('should efficiently filter large result sets', async () => {
  // Create many test documents
  const manyUsers = [];
  for (let i = 0; i < 100; i++) {
    manyUsers.push(new TestUser({
      name: `User ${i}`,
      age: 20 + (i % 40),
      email: `user${i}@example.com`,
      roles: i % 3 === 0 ? ['admin'] : ['user'],
      tags: i % 2 === 0 ? ['even', 'test'] : ['odd', 'test'],
      status: i % 4 === 0 ? 'inactive' : 'active',
      metadata: { loginCount: i },
      scores: [i, i + 10, i + 20]
    }));
  }

  // Save in batches for efficiency
  for (const user of manyUsers) {
    await user.save();
  }

  // Complex filter that should still be fast
  const startTime = Date.now();
  const filtered = await TestUser.getInstances({
    $and: [
      { age: { $gte: 30, $lt: 40 } },
      { status: 'active' },
      { tags: { $in: ['even'] } },
      { 'metadata.loginCount': { $gte: 20 } }
    ]
  });
  const duration = Date.now() - startTime;

  console.log(`Complex filter on 100+ documents took ${duration}ms`);
  expect(duration).toBeLessThan(1000); // Should complete in under 1 second
  expect(filtered.length).toBeGreaterThan(0);

  // Clean up
  for (const user of manyUsers) {
    await user.delete();
  }
});

// ============= CLEANUP =============
tap.test('should clean up test database', async () => {
  await testDb.mongoDb.dropDatabase();
  await testDb.close();
  await smartmongoInstance.stop();
});

export default tap.start();