smartai/test/test.research.anthropic.ts

import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartai from '../ts/index.js';
import * as path from 'path';
import { promises as fs } from 'fs';

const testQenv = new qenv.Qenv('./', './.nogit/');

// Helper function to save research results
async function saveResearchResult(testName: string, result: any) {
  const sanitizedName = testName.replace(/[^a-z0-9]/gi, '_').toLowerCase();
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const filename = `${sanitizedName}_${timestamp}.json`;
  const filepath = path.join('.nogit', 'testresults', 'research', filename);

  await fs.mkdir(path.dirname(filepath), { recursive: true });
  await fs.writeFile(filepath, JSON.stringify(result, null, 2), 'utf-8');

  console.log(`  💾 Saved to: ${filepath}`);
}

let anthropicProvider: smartai.AnthropicProvider;

tap.test('Anthropic Research: should initialize provider with web search', async () => {
  anthropicProvider = new smartai.AnthropicProvider({
    anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
    enableWebSearch: true
  });

  await anthropicProvider.start();
  expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
  expect(typeof anthropicProvider.research).toEqual('function');
});

tap.test('Anthropic Research: should perform basic research query', async () => {
  const result = await anthropicProvider.research({
    query: 'What is machine learning and its main applications?',
    searchDepth: 'basic'
  });

  console.log('Anthropic Basic Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Sources found:', result.sources.length);
  console.log('- First 200 chars:', result.answer.substring(0, 200));

  await saveResearchResult('basic_research_machine_learning', result);

  expect(result).toBeTruthy();
  expect(result.answer).toBeTruthy();
  expect(result.answer.toLowerCase()).toInclude('machine learning');
  expect(result.sources).toBeArray();
  expect(result.metadata).toBeTruthy();
});

tap.test('Anthropic Research: should perform research with web search', async () => {
  const result = await anthropicProvider.research({
    query: 'What are the latest developments in renewable energy technology?',
    searchDepth: 'advanced',
    includeWebSearch: true,
    maxSources: 5
  });

  console.log('Anthropic Web Search Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Sources:', result.sources.length);
  if (result.searchQueries) {
    console.log('- Search queries:', result.searchQueries);
  }

  await saveResearchResult('web_search_renewable_energy', result);

  expect(result.answer).toBeTruthy();
  expect(result.answer.toLowerCase()).toInclude('renewable');

  // Check if sources were extracted
  if (result.sources.length > 0) {
    console.log('- Example source:', result.sources[0]);
    expect(result.sources[0]).toHaveProperty('url');
  }
});

tap.test('Anthropic Research: should handle deep research queries', async () => {
  const result = await anthropicProvider.research({
    query: 'Explain the differences between REST and GraphQL APIs',
    searchDepth: 'deep'
  });

  console.log('Anthropic Deep Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Token usage:', result.metadata?.tokensUsed);

  await saveResearchResult('deep_research_rest_vs_graphql', result);

  expect(result.answer).toBeTruthy();
  expect(result.answer.length).toBeGreaterThan(300);
  expect(result.answer.toLowerCase()).toInclude('rest');
  expect(result.answer.toLowerCase()).toInclude('graphql');
});

tap.test('Anthropic Research: should extract citations from response', async () => {
  const result = await anthropicProvider.research({
    query: 'What is Docker and how does containerization work?',
    searchDepth: 'basic',
    maxSources: 3
  });

  console.log('Anthropic Citation Extraction:');
  console.log('- Sources found:', result.sources.length);
  console.log('- Answer includes Docker:', result.answer.toLowerCase().includes('docker'));

  await saveResearchResult('citation_extraction_docker', result);

  expect(result.answer).toInclude('Docker');

  // Check for URL extraction (both markdown and plain URLs)
  const hasUrls = result.answer.includes('http') || result.sources.length > 0;
  console.log('- Contains URLs or sources:', hasUrls);
});

tap.test('Anthropic Research: should use domain filtering when configured', async () => {
  // Create a new provider with domain restrictions
  const filteredProvider = new smartai.AnthropicProvider({
    anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
    enableWebSearch: true,
    searchDomainAllowList: ['wikipedia.org', 'docs.microsoft.com'],
    searchDomainBlockList: ['reddit.com']
  });

  await filteredProvider.start();

  const result = await filteredProvider.research({
    query: 'What is JavaScript?',
    searchDepth: 'basic'
  });

  console.log('Anthropic Domain Filtering Test:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Applied domain filters (allow: wikipedia, docs.microsoft)');

  await saveResearchResult('domain_filtering_javascript', result);

  expect(result.answer).toBeTruthy();
  expect(result.answer.toLowerCase()).toInclude('javascript');

  await filteredProvider.stop();
});

tap.test('Anthropic Research: should handle errors gracefully', async () => {
  let errorCaught = false;

  try {
    await anthropicProvider.research({
      query: '', // Empty query
      searchDepth: 'basic'
    });
  } catch (error) {
    errorCaught = true;
    console.log('Expected error for empty query:', error.message.substring(0, 100));
  }

  // Anthropic might handle empty queries differently
  console.log(`Empty query error test - Error caught: ${errorCaught}`);
});

tap.test('Anthropic Research: should handle different search depths', async () => {
  // Test basic search depth
  const basicResult = await anthropicProvider.research({
    query: 'What is Python?',
    searchDepth: 'basic'
  });

  // Test advanced search depth
  const advancedResult = await anthropicProvider.research({
    query: 'What is Python?',
    searchDepth: 'advanced'
  });

  console.log('Anthropic Search Depth Comparison:');
  console.log('- Basic answer length:', basicResult.answer.length);
  console.log('- Advanced answer length:', advancedResult.answer.length);
  console.log('- Basic tokens:', basicResult.metadata?.tokensUsed);
  console.log('- Advanced tokens:', advancedResult.metadata?.tokensUsed);

  await saveResearchResult('search_depth_python_basic', basicResult);
  await saveResearchResult('search_depth_python_advanced', advancedResult);

  expect(basicResult.answer).toBeTruthy();
  expect(advancedResult.answer).toBeTruthy();

  // Advanced search typically produces longer answers
  // But this isn't guaranteed, so we just check they exist
  expect(basicResult.answer.toLowerCase()).toInclude('python');
  expect(advancedResult.answer.toLowerCase()).toInclude('python');
});

tap.test('Anthropic Research: ARM vs. Qualcomm comparison', async () => {
  const result = await anthropicProvider.research({
    query: 'Compare ARM and Qualcomm: their technologies, market positions, and recent developments in the mobile and computing sectors',
    searchDepth: 'advanced',
    includeWebSearch: true,
    maxSources: 10
  });

  console.log('ARM vs. Qualcomm Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Sources found:', result.sources.length);
  console.log('- First 300 chars:', result.answer.substring(0, 300));

  await saveResearchResult('arm_vs_qualcomm_comparison', result);

  expect(result.answer).toBeTruthy();
  expect(result.answer.length).toBeGreaterThan(500);
  expect(result.answer.toLowerCase()).toInclude('arm');
  expect(result.answer.toLowerCase()).toInclude('qualcomm');
  expect(result.sources.length).toBeGreaterThan(0);
});

tap.test('Anthropic Research: should clean up provider', async () => {
  await anthropicProvider.stop();
  console.log('Anthropic research provider stopped successfully');
});

export default tap.start();