fix(provider.anthropic): Fix Anthropic research tool identifier and add tests + local Claude permissions
This commit is contained in:
173
test/test.research.anthropic.ts
Normal file
173
test/test.research.anthropic.ts
Normal file
@@ -0,0 +1,173 @@
|
||||
import { expect, tap } from '@push.rocks/tapbundle';
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
import * as smartai from '../ts/index.js';
|
||||
|
||||
const testQenv = new qenv.Qenv('./', './.nogit/');
|
||||
|
||||
let anthropicProvider: smartai.AnthropicProvider;
|
||||
|
||||
tap.test('Anthropic Research: should initialize provider with web search', async () => {
|
||||
anthropicProvider = new smartai.AnthropicProvider({
|
||||
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
|
||||
enableWebSearch: true
|
||||
});
|
||||
|
||||
await anthropicProvider.start();
|
||||
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
|
||||
expect(typeof anthropicProvider.research).toEqual('function');
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should perform basic research query', async () => {
|
||||
const result = await anthropicProvider.research({
|
||||
query: 'What is machine learning and its main applications?',
|
||||
searchDepth: 'basic'
|
||||
});
|
||||
|
||||
console.log('Anthropic Basic Research:');
|
||||
console.log('- Answer length:', result.answer.length);
|
||||
console.log('- Sources found:', result.sources.length);
|
||||
console.log('- First 200 chars:', result.answer.substring(0, 200));
|
||||
|
||||
expect(result).toBeTruthy();
|
||||
expect(result.answer).toBeTruthy();
|
||||
expect(result.answer.toLowerCase()).toInclude('machine learning');
|
||||
expect(result.sources).toBeArray();
|
||||
expect(result.metadata).toBeTruthy();
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should perform research with web search', async () => {
|
||||
const result = await anthropicProvider.research({
|
||||
query: 'What are the latest developments in renewable energy technology?',
|
||||
searchDepth: 'advanced',
|
||||
includeWebSearch: true,
|
||||
maxSources: 5
|
||||
});
|
||||
|
||||
console.log('Anthropic Web Search Research:');
|
||||
console.log('- Answer length:', result.answer.length);
|
||||
console.log('- Sources:', result.sources.length);
|
||||
if (result.searchQueries) {
|
||||
console.log('- Search queries:', result.searchQueries);
|
||||
}
|
||||
|
||||
expect(result.answer).toBeTruthy();
|
||||
expect(result.answer.toLowerCase()).toInclude('renewable');
|
||||
|
||||
// Check if sources were extracted
|
||||
if (result.sources.length > 0) {
|
||||
console.log('- Example source:', result.sources[0]);
|
||||
expect(result.sources[0]).toHaveProperty('url');
|
||||
}
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should handle deep research queries', async () => {
|
||||
const result = await anthropicProvider.research({
|
||||
query: 'Explain the differences between REST and GraphQL APIs',
|
||||
searchDepth: 'deep'
|
||||
});
|
||||
|
||||
console.log('Anthropic Deep Research:');
|
||||
console.log('- Answer length:', result.answer.length);
|
||||
console.log('- Token usage:', result.metadata?.tokensUsed);
|
||||
|
||||
expect(result.answer).toBeTruthy();
|
||||
expect(result.answer.length).toBeGreaterThan(300);
|
||||
expect(result.answer.toLowerCase()).toInclude('rest');
|
||||
expect(result.answer.toLowerCase()).toInclude('graphql');
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should extract citations from response', async () => {
|
||||
const result = await anthropicProvider.research({
|
||||
query: 'What is Docker and how does containerization work?',
|
||||
searchDepth: 'basic',
|
||||
maxSources: 3
|
||||
});
|
||||
|
||||
console.log('Anthropic Citation Extraction:');
|
||||
console.log('- Sources found:', result.sources.length);
|
||||
console.log('- Answer includes Docker:', result.answer.toLowerCase().includes('docker'));
|
||||
|
||||
expect(result.answer).toInclude('Docker');
|
||||
|
||||
// Check for URL extraction (both markdown and plain URLs)
|
||||
const hasUrls = result.answer.includes('http') || result.sources.length > 0;
|
||||
console.log('- Contains URLs or sources:', hasUrls);
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should use domain filtering when configured', async () => {
|
||||
// Create a new provider with domain restrictions
|
||||
const filteredProvider = new smartai.AnthropicProvider({
|
||||
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
|
||||
enableWebSearch: true,
|
||||
searchDomainAllowList: ['wikipedia.org', 'docs.microsoft.com'],
|
||||
searchDomainBlockList: ['reddit.com']
|
||||
});
|
||||
|
||||
await filteredProvider.start();
|
||||
|
||||
const result = await filteredProvider.research({
|
||||
query: 'What is JavaScript?',
|
||||
searchDepth: 'basic'
|
||||
});
|
||||
|
||||
console.log('Anthropic Domain Filtering Test:');
|
||||
console.log('- Answer length:', result.answer.length);
|
||||
console.log('- Applied domain filters (allow: wikipedia, docs.microsoft)');
|
||||
|
||||
expect(result.answer).toBeTruthy();
|
||||
expect(result.answer.toLowerCase()).toInclude('javascript');
|
||||
|
||||
await filteredProvider.stop();
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should handle errors gracefully', async () => {
|
||||
let errorCaught = false;
|
||||
|
||||
try {
|
||||
await anthropicProvider.research({
|
||||
query: '', // Empty query
|
||||
searchDepth: 'basic'
|
||||
});
|
||||
} catch (error) {
|
||||
errorCaught = true;
|
||||
console.log('Expected error for empty query:', error.message.substring(0, 100));
|
||||
}
|
||||
|
||||
// Anthropic might handle empty queries differently
|
||||
console.log(`Empty query error test - Error caught: ${errorCaught}`);
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should handle different search depths', async () => {
|
||||
// Test basic search depth
|
||||
const basicResult = await anthropicProvider.research({
|
||||
query: 'What is Python?',
|
||||
searchDepth: 'basic'
|
||||
});
|
||||
|
||||
// Test advanced search depth
|
||||
const advancedResult = await anthropicProvider.research({
|
||||
query: 'What is Python?',
|
||||
searchDepth: 'advanced'
|
||||
});
|
||||
|
||||
console.log('Anthropic Search Depth Comparison:');
|
||||
console.log('- Basic answer length:', basicResult.answer.length);
|
||||
console.log('- Advanced answer length:', advancedResult.answer.length);
|
||||
console.log('- Basic tokens:', basicResult.metadata?.tokensUsed);
|
||||
console.log('- Advanced tokens:', advancedResult.metadata?.tokensUsed);
|
||||
|
||||
expect(basicResult.answer).toBeTruthy();
|
||||
expect(advancedResult.answer).toBeTruthy();
|
||||
|
||||
// Advanced search typically produces longer answers
|
||||
// But this isn't guaranteed, so we just check they exist
|
||||
expect(basicResult.answer.toLowerCase()).toInclude('python');
|
||||
expect(advancedResult.answer.toLowerCase()).toInclude('python');
|
||||
});
|
||||
|
||||
tap.test('Anthropic Research: should clean up provider', async () => {
|
||||
await anthropicProvider.stop();
|
||||
console.log('Anthropic research provider stopped successfully');
|
||||
});
|
||||
|
||||
export default tap.start();
|
Reference in New Issue
Block a user