diff --git a/test/test.anthropic.ts b/test/test.anthropic.ts index 3108347..1e1ba24 100644 --- a/test/test.anthropic.ts +++ b/test/test.anthropic.ts @@ -48,20 +48,76 @@ tap.test('Anthropic: should handle message history', async () => { expect(response.message.toLowerCase()).toInclude('claude test'); }); -tap.test('Anthropic: should process vision tasks', async () => { - // Create a simple test image (1x1 red pixel JPEG) - // This is a valid 1x1 JPEG image - const redPixelBase64 = '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCwAA8A/9k='; - const imageBuffer = Buffer.from(redPixelBase64, 'base64'); +tap.test('Anthropic: should analyze coffee image with latte art', async () => { + // Test 1: Coffee image from Unsplash by Dani + const imagePath = './test/testimages/coffee-dani/coffee.jpg'; + console.log(`Loading coffee image from: ${imagePath}`); + + const imageBuffer = await smartfile.fs.toBuffer(imagePath); + console.log(`Image loaded, size: ${imageBuffer.length} bytes`); const result = await anthropicProvider.vision({ image: imageBuffer, - prompt: 'What color is this image? Answer with just the color name.' + prompt: 'Describe this coffee image. What do you see in terms of the cup, foam pattern, and overall composition?' }); - console.log(`Anthropic Vision - Result: ${result}`); + console.log(`Anthropic Vision (Coffee) - Result: ${result}`); expect(result).toBeTruthy(); expect(typeof result).toEqual('string'); + expect(result.toLowerCase()).toInclude('coffee'); + // The image has a heart pattern in the latte art + const mentionsLatte = result.toLowerCase().includes('heart') || + result.toLowerCase().includes('latte') || + result.toLowerCase().includes('foam'); + expect(mentionsLatte).toBeTrue(); +}); + +tap.test('Anthropic: should analyze laptop/workspace image', async () => { + // Test 2: Laptop image from Unsplash by Nicolas Bichon + const imagePath = './test/testimages/laptop-nicolas/laptop.jpg'; + console.log(`Loading laptop image from: ${imagePath}`); + + const imageBuffer = await smartfile.fs.toBuffer(imagePath); + console.log(`Image loaded, size: ${imageBuffer.length} bytes`); + + const result = await anthropicProvider.vision({ + image: imageBuffer, + prompt: 'Describe the technology and workspace setup in this image. What devices and equipment can you see?' + }); + + console.log(`Anthropic Vision (Laptop) - Result: ${result}`); + expect(result).toBeTruthy(); + expect(typeof result).toEqual('string'); + // Should mention laptop, computer, keyboard, or desk + const mentionsTech = result.toLowerCase().includes('laptop') || + result.toLowerCase().includes('computer') || + result.toLowerCase().includes('keyboard') || + result.toLowerCase().includes('desk'); + expect(mentionsTech).toBeTrue(); +}); + +tap.test('Anthropic: should analyze receipt/document image', async () => { + // Test 3: Receipt image from Unsplash by Annie Spratt + const imagePath = './test/testimages/receipt-annie/receipt.jpg'; + console.log(`Loading receipt image from: ${imagePath}`); + + const imageBuffer = await smartfile.fs.toBuffer(imagePath); + console.log(`Image loaded, size: ${imageBuffer.length} bytes`); + + const result = await anthropicProvider.vision({ + image: imageBuffer, + prompt: 'What type of document is this? Can you identify any text or numbers visible in the image?' + }); + + console.log(`Anthropic Vision (Receipt) - Result: ${result}`); + expect(result).toBeTruthy(); + expect(typeof result).toEqual('string'); + // Should mention receipt, document, text, or paper + const mentionsDocument = result.toLowerCase().includes('receipt') || + result.toLowerCase().includes('document') || + result.toLowerCase().includes('text') || + result.toLowerCase().includes('paper'); + expect(mentionsDocument).toBeTrue(); }); tap.test('Anthropic: should document a PDF', async () => { diff --git a/test/test.research.anthropic.ts b/test/test.research.anthropic.ts index 5bfd42f..1aac067 100644 --- a/test/test.research.anthropic.ts +++ b/test/test.research.anthropic.ts @@ -1,9 +1,24 @@ import { expect, tap } from '@push.rocks/tapbundle'; import * as qenv from '@push.rocks/qenv'; import * as smartai from '../ts/index.js'; +import * as path from 'path'; +import { promises as fs } from 'fs'; const testQenv = new qenv.Qenv('./', './.nogit/'); +// Helper function to save research results +async function saveResearchResult(testName: string, result: any) { + const sanitizedName = testName.replace(/[^a-z0-9]/gi, '_').toLowerCase(); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `${sanitizedName}_${timestamp}.json`; + const filepath = path.join('.nogit', 'testresults', 'research', filename); + + await fs.mkdir(path.dirname(filepath), { recursive: true }); + await fs.writeFile(filepath, JSON.stringify(result, null, 2), 'utf-8'); + + console.log(` 💾 Saved to: ${filepath}`); +} + let anthropicProvider: smartai.AnthropicProvider; tap.test('Anthropic Research: should initialize provider with web search', async () => { @@ -28,6 +43,8 @@ tap.test('Anthropic Research: should perform basic research query', async () => console.log('- Sources found:', result.sources.length); console.log('- First 200 chars:', result.answer.substring(0, 200)); + await saveResearchResult('basic_research_machine_learning', result); + expect(result).toBeTruthy(); expect(result.answer).toBeTruthy(); expect(result.answer.toLowerCase()).toInclude('machine learning'); @@ -50,6 +67,8 @@ tap.test('Anthropic Research: should perform research with web search', async () console.log('- Search queries:', result.searchQueries); } + await saveResearchResult('web_search_renewable_energy', result); + expect(result.answer).toBeTruthy(); expect(result.answer.toLowerCase()).toInclude('renewable'); @@ -70,6 +89,8 @@ tap.test('Anthropic Research: should handle deep research queries', async () => console.log('- Answer length:', result.answer.length); console.log('- Token usage:', result.metadata?.tokensUsed); + await saveResearchResult('deep_research_rest_vs_graphql', result); + expect(result.answer).toBeTruthy(); expect(result.answer.length).toBeGreaterThan(300); expect(result.answer.toLowerCase()).toInclude('rest'); @@ -87,6 +108,8 @@ tap.test('Anthropic Research: should extract citations from response', async () console.log('- Sources found:', result.sources.length); console.log('- Answer includes Docker:', result.answer.toLowerCase().includes('docker')); + await saveResearchResult('citation_extraction_docker', result); + expect(result.answer).toInclude('Docker'); // Check for URL extraction (both markdown and plain URLs) @@ -114,6 +137,8 @@ tap.test('Anthropic Research: should use domain filtering when configured', asyn console.log('- Answer length:', result.answer.length); console.log('- Applied domain filters (allow: wikipedia, docs.microsoft)'); + await saveResearchResult('domain_filtering_javascript', result); + expect(result.answer).toBeTruthy(); expect(result.answer.toLowerCase()).toInclude('javascript'); @@ -156,6 +181,9 @@ tap.test('Anthropic Research: should handle different search depths', async () = console.log('- Basic tokens:', basicResult.metadata?.tokensUsed); console.log('- Advanced tokens:', advancedResult.metadata?.tokensUsed); + await saveResearchResult('search_depth_python_basic', basicResult); + await saveResearchResult('search_depth_python_advanced', advancedResult); + expect(basicResult.answer).toBeTruthy(); expect(advancedResult.answer).toBeTruthy(); @@ -165,6 +193,28 @@ tap.test('Anthropic Research: should handle different search depths', async () = expect(advancedResult.answer.toLowerCase()).toInclude('python'); }); +tap.test('Anthropic Research: ARM vs. Qualcomm comparison', async () => { + const result = await anthropicProvider.research({ + query: 'Compare ARM and Qualcomm: their technologies, market positions, and recent developments in the mobile and computing sectors', + searchDepth: 'advanced', + includeWebSearch: true, + maxSources: 10 + }); + + console.log('ARM vs. Qualcomm Research:'); + console.log('- Answer length:', result.answer.length); + console.log('- Sources found:', result.sources.length); + console.log('- First 300 chars:', result.answer.substring(0, 300)); + + await saveResearchResult('arm_vs_qualcomm_comparison', result); + + expect(result.answer).toBeTruthy(); + expect(result.answer.length).toBeGreaterThan(500); + expect(result.answer.toLowerCase()).toInclude('arm'); + expect(result.answer.toLowerCase()).toInclude('qualcomm'); + expect(result.sources.length).toBeGreaterThan(0); +}); + tap.test('Anthropic Research: should clean up provider', async () => { await anthropicProvider.stop(); console.log('Anthropic research provider stopped successfully'); diff --git a/test/test.research.openai.ts b/test/test.research.openai.ts index e07bf11..26361ff 100644 --- a/test/test.research.openai.ts +++ b/test/test.research.openai.ts @@ -1,9 +1,24 @@ import { expect, tap } from '@push.rocks/tapbundle'; import * as qenv from '@push.rocks/qenv'; import * as smartai from '../ts/index.js'; +import * as path from 'path'; +import { promises as fs } from 'fs'; const testQenv = new qenv.Qenv('./', './.nogit/'); +// Helper function to save research results +async function saveResearchResult(testName: string, result: any) { + const sanitizedName = testName.replace(/[^a-z0-9]/gi, '_').toLowerCase(); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `openai_${sanitizedName}_${timestamp}.json`; + const filepath = path.join('.nogit', 'testresults', 'research', filename); + + await fs.mkdir(path.dirname(filepath), { recursive: true }); + await fs.writeFile(filepath, JSON.stringify(result, null, 2), 'utf-8'); + + console.log(` 💾 Saved to: ${filepath}`); +} + let openaiProvider: smartai.OpenAiProvider; tap.test('OpenAI Research: should initialize provider with research capabilities', async () => { @@ -29,6 +44,8 @@ tap.test('OpenAI Research: should perform basic research query', async () => { console.log('- Sources found:', result.sources.length); console.log('- First 200 chars:', result.answer.substring(0, 200)); + await saveResearchResult('basic_research_typescript', result); + expect(result).toBeTruthy(); expect(result.answer).toBeTruthy(); expect(result.answer.toLowerCase()).toInclude('typescript'); @@ -52,6 +69,8 @@ tap.test('OpenAI Research: should perform research with web search enabled', asy console.log('- Search queries used:', result.searchQueries); } + await saveResearchResult('web_search_ecmascript', result); + expect(result.answer).toBeTruthy(); expect(result.answer.toLowerCase()).toInclude('ecmascript'); @@ -98,6 +117,8 @@ tap.test('OpenAI Research: should extract sources from markdown links', async () console.log('OpenAI Source Extraction:'); console.log('- Sources found:', result.sources.length); + await saveResearchResult('source_extraction_nodejs', result); + if (result.sources.length > 0) { console.log('- Example source:', result.sources[0]); expect(result.sources[0].url).toBeTruthy(); diff --git a/test/testimages/coffee-dani/README.md b/test/testimages/coffee-dani/README.md new file mode 100644 index 0000000..7303681 --- /dev/null +++ b/test/testimages/coffee-dani/README.md @@ -0,0 +1,36 @@ +# Coffee Image Attribution + +## coffee.jpg + +**Photographer:** Dani (@frokz) +**Source URL:** https://unsplash.com/photos/cup-of-coffee-on-saucer-ZLqxSzvVr7I +**Direct Link:** https://images.unsplash.com/photo-1506372023823-741c83b836fe + +### Metadata +- **Title:** Cup of coffee on saucer +- **Description:** One of many coffee-moments in my life ;) +- **Date Published:** September 25, 2017 +- **Location:** Stockholm, Sweden +- **Tags:** coffee, cafe, heart, coffee cup, cup, barista, latte, mug, saucer, food, sweden, stockholm + +### License +**Unsplash License** - Free to use +- ✅ Commercial and non-commercial use +- ✅ No permission needed +- ❌ Cannot be sold without significant modification +- ❌ Cannot be used to replicate Unsplash or similar service + +Full license: https://unsplash.com/license + +### Usage in This Project +This image is used for testing vision/image processing capabilities in the SmartAI library test suite, specifically for: +- Testing coffee/beverage recognition +- Latte art pattern detection (heart shape) +- Scene/environment analysis +- Multi-element image understanding (cup, saucer, table) + +### Download Information +- **Downloaded:** September 28, 2025 +- **Original Filename:** dani-ZLqxSzvVr7I-unsplash.jpg +- **Resolution:** High resolution (3.7 MB) +- **Format:** JPEG \ No newline at end of file diff --git a/test/testimages/coffee-dani/coffee.jpg b/test/testimages/coffee-dani/coffee.jpg new file mode 100644 index 0000000..5276a80 Binary files /dev/null and b/test/testimages/coffee-dani/coffee.jpg differ diff --git a/test/testimages/laptop-nicolas/README.md b/test/testimages/laptop-nicolas/README.md new file mode 100644 index 0000000..2226250 --- /dev/null +++ b/test/testimages/laptop-nicolas/README.md @@ -0,0 +1,40 @@ +# Laptop Image Attribution + +## laptop.jpg + +**Photographer:** Nicolas Bichon (@nicol3a) +**Source URL:** https://unsplash.com/photos/a-laptop-computer-sitting-on-top-of-a-wooden-desk-ZhV4iqAXxyA +**Direct Link:** https://images.unsplash.com/photo-1704230972797-e0e3aba0fce7 + +### Metadata +- **Title:** A laptop computer sitting on top of a wooden desk +- **Description:** Lifestyle photo I took for my indie app Type, a macOS app to take notes without interrupting your flow. https://usetype.app. +- **Date Published:** January 2, 2024 +- **Camera:** FUJIFILM, X-T20 +- **Tags:** computer, laptop, mac, keyboard, computer keyboard, computer hardware, furniture, table, electronics, screen, monitor, hardware, display, tabletop, lcd screen, digital display + +### Statistics +- **Views:** 183,020 +- **Downloads:** 757 + +### License +**Unsplash License** - Free to use +- ✅ Commercial and non-commercial use +- ✅ No permission needed +- ❌ Cannot be sold without significant modification +- ❌ Cannot be used to replicate Unsplash or similar service + +Full license: https://unsplash.com/license + +### Usage in This Project +This image is used for testing vision/image processing capabilities in the SmartAI library test suite, specifically for: +- Testing technology/computer equipment recognition +- Workspace/office environment analysis +- Object detection (laptop, keyboard, monitor, table) +- Scene understanding and context analysis + +### Download Information +- **Downloaded:** September 28, 2025 +- **Original Filename:** nicolas-bichon-ZhV4iqAXxyA-unsplash.jpg +- **Resolution:** High resolution (1.8 MB) +- **Format:** JPEG \ No newline at end of file diff --git a/test/testimages/laptop-nicolas/laptop.jpg b/test/testimages/laptop-nicolas/laptop.jpg new file mode 100644 index 0000000..e3e7a3d Binary files /dev/null and b/test/testimages/laptop-nicolas/laptop.jpg differ diff --git a/test/testimages/receipt-annie/README.md b/test/testimages/receipt-annie/README.md new file mode 100644 index 0000000..569cd64 --- /dev/null +++ b/test/testimages/receipt-annie/README.md @@ -0,0 +1,40 @@ +# Receipt Image Attribution + +## receipt.jpg + +**Photographer:** Annie Spratt (@anniespratt) +**Source URL:** https://unsplash.com/photos/a-receipt-sitting-on-top-of-a-wooden-table-recgFWxDO1Y +**Direct Link:** https://images.unsplash.com/photo-1731686602391-7484df33a03c + +### Metadata +- **Title:** A receipt sitting on top of a wooden table +- **Description:** Download this free HD photo of text, document, invoice, and receipt by Annie Spratt +- **Date Published:** November 15, 2024 +- **Tags:** text, document, invoice, receipt, diaper + +### Statistics +- **Views:** 54,593 +- **Downloads:** 764 + +### License +**Unsplash License** - Free to use +- ✅ Commercial and non-commercial use +- ✅ No permission needed +- ❌ Cannot be sold without significant modification +- ❌ Cannot be used to replicate Unsplash or similar service + +Full license: https://unsplash.com/license + +### Usage in This Project +This image is used for testing vision/image processing capabilities in the SmartAI library test suite, specifically for: +- Testing text extraction and OCR capabilities +- Document recognition and classification +- Receipt/invoice analysis +- Text-heavy image understanding +- Structured data extraction from documents + +### Download Information +- **Downloaded:** September 28, 2025 +- **Original Filename:** annie-spratt-recgFWxDO1Y-unsplash.jpg +- **Resolution:** High resolution (3.3 MB) +- **Format:** JPEG \ No newline at end of file diff --git a/test/testimages/receipt-annie/receipt.jpg b/test/testimages/receipt-annie/receipt.jpg new file mode 100644 index 0000000..07d3848 Binary files /dev/null and b/test/testimages/receipt-annie/receipt.jpg differ diff --git a/ts/provider.anthropic.ts b/ts/provider.anthropic.ts index 0284e13..74a78c4 100644 --- a/ts/provider.anthropic.ts +++ b/ts/provider.anthropic.ts @@ -68,7 +68,7 @@ export class AnthropicProvider extends MultiModalModel { // If we have a complete message, send it to Anthropic if (currentMessage) { const stream = await this.anthropicApiClient.messages.create({ - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', messages: [{ role: currentMessage.role, content: currentMessage.content }], system: '', stream: true, @@ -112,7 +112,7 @@ export class AnthropicProvider extends MultiModalModel { })); const result = await this.anthropicApiClient.messages.create({ - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', system: optionsArg.systemMessage, messages: [ ...messages, @@ -159,7 +159,7 @@ export class AnthropicProvider extends MultiModalModel { ]; const result = await this.anthropicApiClient.messages.create({ - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', messages: [{ role: 'user', content @@ -218,7 +218,7 @@ export class AnthropicProvider extends MultiModalModel { } const result = await this.anthropicApiClient.messages.create({ - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', system: optionsArg.systemMessage, messages: [ ...messages, @@ -251,23 +251,27 @@ export class AnthropicProvider extends MultiModalModel { try { // Build the tool configuration for web search - const tools = this.options.enableWebSearch ? [ - { - type: 'web_search_20250305' as const, - name: 'web_search', - description: 'Search the web for current information', - input_schema: { - type: 'object' as const, - properties: { - query: { - type: 'string', - description: 'The search query' - } - }, - required: ['query'] - } + const tools: any[] = []; + + if (this.options.enableWebSearch) { + const webSearchTool: any = { + type: 'web_search_20250305', + name: 'web_search' + }; + + // Add optional parameters + if (optionsArg.maxSources) { + webSearchTool.max_uses = optionsArg.maxSources; } - ] : []; + + if (this.options.searchDomainAllowList?.length) { + webSearchTool.allowed_domains = this.options.searchDomainAllowList; + } else if (this.options.searchDomainBlockList?.length) { + webSearchTool.blocked_domains = this.options.searchDomainBlockList; + } + + tools.push(webSearchTool); + } // Configure the request based on search depth const maxTokens = optionsArg.searchDepth === 'deep' ? 8192 : @@ -275,7 +279,7 @@ export class AnthropicProvider extends MultiModalModel { // Create the research request const requestParams: any = { - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', system: systemMessage, messages: [ { @@ -290,7 +294,6 @@ export class AnthropicProvider extends MultiModalModel { // Add tools if web search is enabled if (tools.length > 0) { requestParams.tools = tools; - requestParams.tool_choice = { type: 'auto' }; } // Execute the research request @@ -304,54 +307,71 @@ export class AnthropicProvider extends MultiModalModel { // Process content blocks for (const block of result.content) { if ('text' in block) { + // Accumulate text content answer += block.text; + + // Extract citations if present + if ('citations' in block && Array.isArray(block.citations)) { + for (const citation of block.citations) { + if (citation.type === 'web_search_result_location') { + sources.push({ + title: citation.title || '', + url: citation.url || '', + snippet: citation.cited_text || '' + }); + } + } + } + } else if ('type' in block && block.type === 'server_tool_use') { + // Extract search queries from server tool use + if (block.name === 'web_search' && block.input && typeof block.input === 'object' && 'query' in block.input) { + searchQueries.push((block.input as any).query); + } + } else if ('type' in block && block.type === 'web_search_tool_result') { + // Extract sources from web search results + if (Array.isArray(block.content)) { + for (const result of block.content) { + if (result.type === 'web_search_result') { + // Only add if not already in sources (avoid duplicates from citations) + if (!sources.some(s => s.url === result.url)) { + sources.push({ + title: result.title || '', + url: result.url || '', + snippet: '' // Search results don't include snippets, only citations do + }); + } + } + } + } } } - // Parse sources from the answer (Claude includes citations in various formats) - const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g; - let match: RegExpExecArray | null; + // Fallback: Parse markdown-style links if no citations found + if (sources.length === 0) { + const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g; + let match: RegExpExecArray | null; - while ((match = urlRegex.exec(answer)) !== null) { - sources.push({ - title: match[1], - url: match[2], - snippet: '' - }); - } - - // Also look for plain URLs - const plainUrlRegex = /https?:\/\/[^\s\)]+/g; - const plainUrls = answer.match(plainUrlRegex) || []; - - for (const url of plainUrls) { - // Check if this URL is already in sources - if (!sources.some(s => s.url === url)) { + while ((match = urlRegex.exec(answer)) !== null) { sources.push({ - title: new URL(url).hostname, - url: url, + title: match[1], + url: match[2], snippet: '' }); } } - // Extract tool use information if available - if ('tool_use' in result && Array.isArray(result.tool_use)) { - for (const toolUse of result.tool_use) { - if (toolUse.name === 'web_search' && toolUse.input?.query) { - searchQueries.push(toolUse.input.query); - } - } - } + // Check if web search was used based on usage info + const webSearchCount = result.usage?.server_tool_use?.web_search_requests || 0; return { answer, sources, searchQueries: searchQueries.length > 0 ? searchQueries : undefined, metadata: { - model: 'claude-3-opus-20240229', + model: 'claude-sonnet-4-5-20250929', searchDepth: optionsArg.searchDepth || 'basic', - tokensUsed: result.usage?.output_tokens + tokensUsed: result.usage?.output_tokens, + webSearchesPerformed: webSearchCount } }; } catch (error) { diff --git a/ts/provider.openai.ts b/ts/provider.openai.ts index 14c6d0c..4c2f28b 100644 --- a/ts/provider.openai.ts +++ b/ts/provider.openai.ts @@ -233,52 +233,37 @@ export class OpenAiProvider extends MultiModalModel { } public async research(optionsArg: ResearchOptions): Promise { - // Determine which model to use based on search depth + // Determine which model to use - Deep Research API requires specific models let model: string; if (optionsArg.searchDepth === 'deep') { model = this.options.researchModel || 'o4-mini-deep-research-2025-06-26'; } else { - model = this.options.chatModel || 'gpt-5-mini'; + // For basic/advanced, still use deep research models if web search is needed + if (optionsArg.includeWebSearch) { + model = this.options.researchModel || 'o4-mini-deep-research-2025-06-26'; + } else { + model = this.options.chatModel || 'gpt-5-mini'; + } } - // Prepare the request parameters + const systemMessage = 'You are a research assistant. Provide comprehensive answers with citations and sources when available.'; + + // Prepare request parameters using Deep Research API format const requestParams: any = { model, - messages: [ - { - role: 'system', - content: 'You are a research assistant. Provide comprehensive answers with citations and sources when available.' - }, - { - role: 'user', - content: optionsArg.query - } - ], - temperature: 0.7 + instructions: systemMessage, + input: optionsArg.query }; - // Add web search tools if requested + // Add web search tool if requested if (optionsArg.includeWebSearch || optionsArg.searchDepth === 'deep') { requestParams.tools = [ { - type: 'function', - function: { - name: 'web_search', - description: 'Search the web for information', - parameters: { - type: 'object', - properties: { - query: { - type: 'string', - description: 'The search query' - } - }, - required: ['query'] - } - } + type: 'web_search_preview', + search_context_size: optionsArg.searchDepth === 'deep' ? 'high' : + optionsArg.searchDepth === 'advanced' ? 'medium' : 'low' } ]; - requestParams.tool_choice = 'auto'; } // Add background flag for deep research @@ -287,14 +272,36 @@ export class OpenAiProvider extends MultiModalModel { } try { - // Execute the research request - const result = await this.openAiApiClient.chat.completions.create(requestParams); + // Execute the research request using Deep Research API + const result = await this.openAiApiClient.responses.create(requestParams); - // Extract the answer - const answer = result.choices[0].message.content || ''; - - // Parse sources from the response (OpenAI often includes URLs in markdown format) + // Extract the answer from output items + let answer = ''; const sources: Array<{ url: string; title: string; snippet: string }> = []; + const searchQueries: string[] = []; + + // Process output items + for (const item of result.output || []) { + // Extract message content + if (item.type === 'message' && 'content' in item) { + const messageItem = item as any; + for (const contentItem of messageItem.content || []) { + if (contentItem.type === 'output_text' && 'text' in contentItem) { + answer += contentItem.text; + } + } + } + + // Extract web search queries + if (item.type === 'web_search_call' && 'action' in item) { + const searchItem = item as any; + if (searchItem.action && searchItem.action.type === 'search' && 'query' in searchItem.action) { + searchQueries.push(searchItem.action.query); + } + } + } + + // Parse sources from markdown links in the answer const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g; let match: RegExpExecArray | null; @@ -302,27 +309,10 @@ export class OpenAiProvider extends MultiModalModel { sources.push({ title: match[1], url: match[2], - snippet: '' // OpenAI doesn't provide snippets in standard responses + snippet: '' }); } - // Extract search queries if tools were used - const searchQueries: string[] = []; - if (result.choices[0].message.tool_calls) { - for (const toolCall of result.choices[0].message.tool_calls) { - if ('function' in toolCall && toolCall.function.name === 'web_search') { - try { - const args = JSON.parse(toolCall.function.arguments); - if (args.query) { - searchQueries.push(args.query); - } - } catch (e) { - // Ignore parsing errors - } - } - } - } - return { answer, sources,