smartai/test/test.research.openai.ts

import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartai from '../ts/index.js';
import * as path from 'path';
import { promises as fs } from 'fs';

const testQenv = new qenv.Qenv('./', './.nogit/');

// Helper function to save research results
async function saveResearchResult(testName: string, result: any) {
  const sanitizedName = testName.replace(/[^a-z0-9]/gi, '_').toLowerCase();
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const filename = `openai_${sanitizedName}_${timestamp}.json`;
  const filepath = path.join('.nogit', 'testresults', 'research', filename);

  await fs.mkdir(path.dirname(filepath), { recursive: true });
  await fs.writeFile(filepath, JSON.stringify(result, null, 2), 'utf-8');

  console.log(`  💾 Saved to: ${filepath}`);
}

let openaiProvider: smartai.OpenAiProvider;

tap.test('OpenAI Research: should initialize provider with research capabilities', async () => {
  openaiProvider = new smartai.OpenAiProvider({
    openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
    researchModel: 'o4-mini-deep-research-2025-06-26',
    enableWebSearch: true
  });

  await openaiProvider.start();
  expect(openaiProvider).toBeInstanceOf(smartai.OpenAiProvider);
  expect(typeof openaiProvider.research).toEqual('function');
});

tap.test('OpenAI Research: should perform basic research query', async () => {
  const result = await openaiProvider.research({
    query: 'What is TypeScript and why is it useful for web development?',
    searchDepth: 'basic'
  });

  console.log('OpenAI Basic Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Sources found:', result.sources.length);
  console.log('- First 200 chars:', result.answer.substring(0, 200));

  await saveResearchResult('basic_research_typescript', result);

  expect(result).toBeTruthy();
  expect(result.answer).toBeTruthy();
  expect(result.answer.toLowerCase()).toInclude('typescript');
  expect(result.sources).toBeArray();
  expect(result.metadata).toBeTruthy();
  expect(result.metadata.model).toBeTruthy();
});

tap.test('OpenAI Research: should perform research with web search enabled', async () => {
  const result = await openaiProvider.research({
    query: 'What are the latest features in ECMAScript 2024?',
    searchDepth: 'advanced',
    includeWebSearch: true,
    maxSources: 5
  });

  console.log('OpenAI Web Search Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Sources:', result.sources.length);
  if (result.searchQueries) {
    console.log('- Search queries used:', result.searchQueries);
  }

  await saveResearchResult('web_search_ecmascript', result);

  expect(result.answer).toBeTruthy();
  expect(result.answer.toLowerCase()).toInclude('ecmascript');

  // The model might include sources or search queries
  if (result.sources.length > 0) {
    expect(result.sources[0]).toHaveProperty('url');
    expect(result.sources[0]).toHaveProperty('title');
  }
});

tap.test('OpenAI Research: should handle deep research for complex topics', async () => {
  // Skip this test if it takes too long or costs too much
  // You can enable it for thorough testing
  const skipDeepResearch = true;

  if (skipDeepResearch) {
    console.log('Skipping deep research test to save API costs');
    return;
  }

  const result = await openaiProvider.research({
    query: 'Compare the pros and cons of microservices vs monolithic architecture',
    searchDepth: 'deep',
    includeWebSearch: true
  });

  console.log('OpenAI Deep Research:');
  console.log('- Answer length:', result.answer.length);
  console.log('- Token usage:', result.metadata?.tokensUsed);

  expect(result.answer).toBeTruthy();
  expect(result.answer.length).toBeGreaterThan(500);
  expect(result.answer.toLowerCase()).toInclude('microservices');
  expect(result.answer.toLowerCase()).toInclude('monolithic');
});

tap.test('OpenAI Research: should extract sources from markdown links', async () => {
  const result = await openaiProvider.research({
    query: 'What is Node.js and provide some official documentation links?',
    searchDepth: 'basic',
    maxSources: 3
  });

  console.log('OpenAI Source Extraction:');
  console.log('- Sources found:', result.sources.length);

  await saveResearchResult('source_extraction_nodejs', result);

  if (result.sources.length > 0) {
    console.log('- Example source:', result.sources[0]);
    expect(result.sources[0].url).toBeTruthy();
    expect(result.sources[0].title).toBeTruthy();
  }

  expect(result.answer).toInclude('Node.js');
});

tap.test('OpenAI Research: should handle research errors gracefully', async () => {
  // Test with an extremely long query that might cause issues
  const longQuery = 'a'.repeat(10000);

  let errorCaught = false;
  try {
    await openaiProvider.research({
      query: longQuery,
      searchDepth: 'basic'
    });
  } catch (error) {
    errorCaught = true;
    console.log('Expected error for long query:', error.message.substring(0, 100));
    expect(error.message).toBeTruthy();
  }

  // OpenAI might handle long queries, so we don't assert the error
  console.log(`Long query error test - Error caught: ${errorCaught}`);
});

tap.test('OpenAI Research: should respect maxSources parameter', async () => {
  const maxSources = 3;
  const result = await openaiProvider.research({
    query: 'List popular JavaScript frameworks',
    searchDepth: 'basic',
    maxSources: maxSources
  });

  console.log(`OpenAI Max Sources Test - Requested: ${maxSources}, Found: ${result.sources.length}`);

  // The API might not always return exactly maxSources, but should respect it as a limit
  if (result.sources.length > 0) {
    expect(result.sources.length).toBeLessThanOrEqual(maxSources * 2); // Allow some flexibility
  }
});

tap.test('OpenAI Research: should clean up provider', async () => {
  await openaiProvider.stop();
  console.log('OpenAI research provider stopped successfully');
});

export default tap.start();