Compare commits

..

2 Commits

Author SHA1 Message Date
e34bf19698 0.6.1
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-09-28 15:51:50 +00:00
f70353e6ca fix(provider.anthropic): Fix Anthropic research tool identifier and add tests + local Claude permissions 2025-09-28 15:51:50 +00:00
10 changed files with 579 additions and 71 deletions

View File

@@ -1,5 +1,13 @@
# Changelog
## 2025-09-28 - 0.6.1 - fix(provider.anthropic)
Fix Anthropic research tool identifier and add tests + local Claude permissions
- Replace Anthropic research tool type from 'computer_20241022' to 'web_search_20250305' to match the expected web-search tool schema.
- Add comprehensive test suites and fixtures for providers and research features (new/updated tests under test/ including anthropic, openai, research.* and stubs).
- Fix test usage of XAI provider class name (use XAIProvider) and adjust basic provider test expectations (provider instantiation moved to start()).
- Add .claude/settings.local.json with local Claude permissions to allow common CI/dev commands and web search during testing.
## 2025-09-28 - 0.6.0 - feat(research)
Introduce research API with provider implementations, docs and tests

View File

@@ -1,6 +1,6 @@
{
"name": "@push.rocks/smartai",
"version": "0.6.0",
"version": "0.6.1",
"private": false,
"description": "SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.",
"main": "dist_ts/index.js",

160
test/test.anthropic.ts Normal file
View File

@@ -0,0 +1,160 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic: should create and start Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
});
tap.test('Anthropic: should create chat response', async () => {
const userMessage = 'What is the capital of France? Answer in one word.';
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant. Be concise.',
userMessage: userMessage,
messageHistory: [],
});
console.log(`Anthropic Chat - User: ${userMessage}`);
console.log(`Anthropic Chat - Response: ${response.message}`);
expect(response.role).toEqual('assistant');
expect(response.message).toBeTruthy();
expect(response.message.toLowerCase()).toInclude('paris');
});
tap.test('Anthropic: should handle message history', async () => {
const messageHistory: smartai.ChatMessage[] = [
{ role: 'user', content: 'My name is Claude Test' },
{ role: 'assistant', content: 'Nice to meet you, Claude Test!' }
];
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant with good memory.',
userMessage: 'What is my name?',
messageHistory: messageHistory,
});
console.log(`Anthropic Memory Test - Response: ${response.message}`);
expect(response.message.toLowerCase()).toInclude('claude test');
});
tap.test('Anthropic: should process vision tasks', async () => {
// Create a simple test image (1x1 red pixel JPEG)
// This is a valid 1x1 JPEG image
const redPixelBase64 = '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCwAA8A/9k=';
const imageBuffer = Buffer.from(redPixelBase64, 'base64');
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'What color is this image? Answer with just the color name.'
});
console.log(`Anthropic Vision - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
});
tap.test('Anthropic: should document a PDF', async () => {
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
const result = await anthropicProvider.document({
systemMessage: 'Classify the document. Only the following answers are allowed: "invoice", "bank account statement", "contract", "test document", "other". The answer should only contain the keyword for machine use.',
userMessage: 'Classify this document.',
messageHistory: [],
pdfDocuments: [Buffer.from(await pdfResponse.arrayBuffer())],
});
console.log(`Anthropic Document - Result:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic: should handle complex document analysis', async () => {
// Test with the demo PDF if it exists
const pdfPath = './.nogit/demo_without_textlayer.pdf';
let pdfBuffer: Uint8Array;
try {
pdfBuffer = await smartfile.fs.toBuffer(pdfPath);
} catch (error) {
// If the file doesn't exist, use the dummy PDF
console.log('Demo PDF not found, using dummy PDF instead');
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
pdfBuffer = Buffer.from(await pdfResponse.arrayBuffer());
}
const result = await anthropicProvider.document({
systemMessage: `
Analyze this document and provide a JSON response with the following structure:
{
"documentType": "string",
"hasText": boolean,
"summary": "string"
}
`,
userMessage: 'Analyze this document.',
messageHistory: [],
pdfDocuments: [pdfBuffer],
});
console.log(`Anthropic Complex Document Analysis:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic: should handle errors gracefully', async () => {
// Test with invalid message (empty)
let errorCaught = false;
try {
await anthropicProvider.chat({
systemMessage: '',
userMessage: '',
messageHistory: [],
});
} catch (error) {
errorCaught = true;
console.log('Expected error caught:', error.message);
}
// Anthropic might handle empty messages, so we don't assert error
console.log(`Error handling test - Error caught: ${errorCaught}`);
});
tap.test('Anthropic: audio should throw not supported error', async () => {
let errorCaught = false;
try {
await anthropicProvider.audio({
message: 'This should fail'
});
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Anthropic: should stop the provider', async () => {
await anthropicProvider.stop();
console.log('Anthropic provider stopped successfully');
});
export default tap.start();

View File

@@ -9,7 +9,8 @@ tap.test('Basic: should create SmartAi instance', async () => {
openaiToken: 'dummy-token-for-testing'
});
expect(testSmartai).toBeInstanceOf(smartai.SmartAi);
expect(testSmartai.openaiProvider).toBeTruthy();
// Provider is only created after calling start()
expect(testSmartai.options.openaiToken).toEqual('dummy-token-for-testing');
});
tap.test('Basic: should instantiate OpenAI provider', async () => {
@@ -64,10 +65,10 @@ tap.test('Basic: should instantiate Ollama provider', async () => {
});
tap.test('Basic: should instantiate xAI provider', async () => {
const xaiProvider = new smartai.XaiProvider({
const xaiProvider = new smartai.XAIProvider({
xaiToken: 'dummy-token'
});
expect(xaiProvider).toBeInstanceOf(smartai.XaiProvider);
expect(xaiProvider).toBeInstanceOf(smartai.XAIProvider);
expect(typeof xaiProvider.chat).toEqual('function');
expect(typeof xaiProvider.research).toEqual('function');
});

View File

@@ -0,0 +1,173 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartai from '../ts/index.js';
const testQenv = new qenv.Qenv('./', './.nogit/');
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic Research: should initialize provider with web search', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
enableWebSearch: true
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
expect(typeof anthropicProvider.research).toEqual('function');
});
tap.test('Anthropic Research: should perform basic research query', async () => {
const result = await anthropicProvider.research({
query: 'What is machine learning and its main applications?',
searchDepth: 'basic'
});
console.log('Anthropic Basic Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Sources found:', result.sources.length);
console.log('- First 200 chars:', result.answer.substring(0, 200));
expect(result).toBeTruthy();
expect(result.answer).toBeTruthy();
expect(result.answer.toLowerCase()).toInclude('machine learning');
expect(result.sources).toBeArray();
expect(result.metadata).toBeTruthy();
});
tap.test('Anthropic Research: should perform research with web search', async () => {
const result = await anthropicProvider.research({
query: 'What are the latest developments in renewable energy technology?',
searchDepth: 'advanced',
includeWebSearch: true,
maxSources: 5
});
console.log('Anthropic Web Search Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Sources:', result.sources.length);
if (result.searchQueries) {
console.log('- Search queries:', result.searchQueries);
}
expect(result.answer).toBeTruthy();
expect(result.answer.toLowerCase()).toInclude('renewable');
// Check if sources were extracted
if (result.sources.length > 0) {
console.log('- Example source:', result.sources[0]);
expect(result.sources[0]).toHaveProperty('url');
}
});
tap.test('Anthropic Research: should handle deep research queries', async () => {
const result = await anthropicProvider.research({
query: 'Explain the differences between REST and GraphQL APIs',
searchDepth: 'deep'
});
console.log('Anthropic Deep Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Token usage:', result.metadata?.tokensUsed);
expect(result.answer).toBeTruthy();
expect(result.answer.length).toBeGreaterThan(300);
expect(result.answer.toLowerCase()).toInclude('rest');
expect(result.answer.toLowerCase()).toInclude('graphql');
});
tap.test('Anthropic Research: should extract citations from response', async () => {
const result = await anthropicProvider.research({
query: 'What is Docker and how does containerization work?',
searchDepth: 'basic',
maxSources: 3
});
console.log('Anthropic Citation Extraction:');
console.log('- Sources found:', result.sources.length);
console.log('- Answer includes Docker:', result.answer.toLowerCase().includes('docker'));
expect(result.answer).toInclude('Docker');
// Check for URL extraction (both markdown and plain URLs)
const hasUrls = result.answer.includes('http') || result.sources.length > 0;
console.log('- Contains URLs or sources:', hasUrls);
});
tap.test('Anthropic Research: should use domain filtering when configured', async () => {
// Create a new provider with domain restrictions
const filteredProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
enableWebSearch: true,
searchDomainAllowList: ['wikipedia.org', 'docs.microsoft.com'],
searchDomainBlockList: ['reddit.com']
});
await filteredProvider.start();
const result = await filteredProvider.research({
query: 'What is JavaScript?',
searchDepth: 'basic'
});
console.log('Anthropic Domain Filtering Test:');
console.log('- Answer length:', result.answer.length);
console.log('- Applied domain filters (allow: wikipedia, docs.microsoft)');
expect(result.answer).toBeTruthy();
expect(result.answer.toLowerCase()).toInclude('javascript');
await filteredProvider.stop();
});
tap.test('Anthropic Research: should handle errors gracefully', async () => {
let errorCaught = false;
try {
await anthropicProvider.research({
query: '', // Empty query
searchDepth: 'basic'
});
} catch (error) {
errorCaught = true;
console.log('Expected error for empty query:', error.message.substring(0, 100));
}
// Anthropic might handle empty queries differently
console.log(`Empty query error test - Error caught: ${errorCaught}`);
});
tap.test('Anthropic Research: should handle different search depths', async () => {
// Test basic search depth
const basicResult = await anthropicProvider.research({
query: 'What is Python?',
searchDepth: 'basic'
});
// Test advanced search depth
const advancedResult = await anthropicProvider.research({
query: 'What is Python?',
searchDepth: 'advanced'
});
console.log('Anthropic Search Depth Comparison:');
console.log('- Basic answer length:', basicResult.answer.length);
console.log('- Advanced answer length:', advancedResult.answer.length);
console.log('- Basic tokens:', basicResult.metadata?.tokensUsed);
console.log('- Advanced tokens:', advancedResult.metadata?.tokensUsed);
expect(basicResult.answer).toBeTruthy();
expect(advancedResult.answer).toBeTruthy();
// Advanced search typically produces longer answers
// But this isn't guaranteed, so we just check they exist
expect(basicResult.answer.toLowerCase()).toInclude('python');
expect(advancedResult.answer.toLowerCase()).toInclude('python');
});
tap.test('Anthropic Research: should clean up provider', async () => {
await anthropicProvider.stop();
console.log('Anthropic research provider stopped successfully');
});
export default tap.start();

View File

@@ -0,0 +1,151 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartai from '../ts/index.js';
const testQenv = new qenv.Qenv('./', './.nogit/');
let openaiProvider: smartai.OpenAiProvider;
tap.test('OpenAI Research: should initialize provider with research capabilities', async () => {
openaiProvider = new smartai.OpenAiProvider({
openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
researchModel: 'o4-mini-deep-research-2025-06-26',
enableWebSearch: true
});
await openaiProvider.start();
expect(openaiProvider).toBeInstanceOf(smartai.OpenAiProvider);
expect(typeof openaiProvider.research).toEqual('function');
});
tap.test('OpenAI Research: should perform basic research query', async () => {
const result = await openaiProvider.research({
query: 'What is TypeScript and why is it useful for web development?',
searchDepth: 'basic'
});
console.log('OpenAI Basic Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Sources found:', result.sources.length);
console.log('- First 200 chars:', result.answer.substring(0, 200));
expect(result).toBeTruthy();
expect(result.answer).toBeTruthy();
expect(result.answer.toLowerCase()).toInclude('typescript');
expect(result.sources).toBeArray();
expect(result.metadata).toBeTruthy();
expect(result.metadata.model).toBeTruthy();
});
tap.test('OpenAI Research: should perform research with web search enabled', async () => {
const result = await openaiProvider.research({
query: 'What are the latest features in ECMAScript 2024?',
searchDepth: 'advanced',
includeWebSearch: true,
maxSources: 5
});
console.log('OpenAI Web Search Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Sources:', result.sources.length);
if (result.searchQueries) {
console.log('- Search queries used:', result.searchQueries);
}
expect(result.answer).toBeTruthy();
expect(result.answer.toLowerCase()).toInclude('ecmascript');
// The model might include sources or search queries
if (result.sources.length > 0) {
expect(result.sources[0]).toHaveProperty('url');
expect(result.sources[0]).toHaveProperty('title');
}
});
tap.test('OpenAI Research: should handle deep research for complex topics', async () => {
// Skip this test if it takes too long or costs too much
// You can enable it for thorough testing
const skipDeepResearch = true;
if (skipDeepResearch) {
console.log('Skipping deep research test to save API costs');
return;
}
const result = await openaiProvider.research({
query: 'Compare the pros and cons of microservices vs monolithic architecture',
searchDepth: 'deep',
includeWebSearch: true
});
console.log('OpenAI Deep Research:');
console.log('- Answer length:', result.answer.length);
console.log('- Token usage:', result.metadata?.tokensUsed);
expect(result.answer).toBeTruthy();
expect(result.answer.length).toBeGreaterThan(500);
expect(result.answer.toLowerCase()).toInclude('microservices');
expect(result.answer.toLowerCase()).toInclude('monolithic');
});
tap.test('OpenAI Research: should extract sources from markdown links', async () => {
const result = await openaiProvider.research({
query: 'What is Node.js and provide some official documentation links?',
searchDepth: 'basic',
maxSources: 3
});
console.log('OpenAI Source Extraction:');
console.log('- Sources found:', result.sources.length);
if (result.sources.length > 0) {
console.log('- Example source:', result.sources[0]);
expect(result.sources[0].url).toBeTruthy();
expect(result.sources[0].title).toBeTruthy();
}
expect(result.answer).toInclude('Node.js');
});
tap.test('OpenAI Research: should handle research errors gracefully', async () => {
// Test with an extremely long query that might cause issues
const longQuery = 'a'.repeat(10000);
let errorCaught = false;
try {
await openaiProvider.research({
query: longQuery,
searchDepth: 'basic'
});
} catch (error) {
errorCaught = true;
console.log('Expected error for long query:', error.message.substring(0, 100));
expect(error.message).toBeTruthy();
}
// OpenAI might handle long queries, so we don't assert the error
console.log(`Long query error test - Error caught: ${errorCaught}`);
});
tap.test('OpenAI Research: should respect maxSources parameter', async () => {
const maxSources = 3;
const result = await openaiProvider.research({
query: 'List popular JavaScript frameworks',
searchDepth: 'basic',
maxSources: maxSources
});
console.log(`OpenAI Max Sources Test - Requested: ${maxSources}, Found: ${result.sources.length}`);
// The API might not always return exactly maxSources, but should respect it as a limit
if (result.sources.length > 0) {
expect(result.sources.length).toBeLessThanOrEqual(maxSources * 2); // Allow some flexibility
}
});
tap.test('OpenAI Research: should clean up provider', async () => {
await openaiProvider.stop();
console.log('OpenAI research provider stopped successfully');
});
export default tap.start();

View File

@@ -0,0 +1,80 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as smartai from '../ts/index.js';
// Test research method stubs for providers without full implementation
// These providers have research methods that throw "not yet supported" errors
tap.test('Research Stubs: Perplexity provider should have research method', async () => {
const perplexityProvider = new smartai.PerplexityProvider({
perplexityToken: 'test-token'
});
// Perplexity has a basic implementation with Sonar models
expect(typeof perplexityProvider.research).toEqual('function');
});
tap.test('Research Stubs: Groq provider should throw not supported error', async () => {
const groqProvider = new smartai.GroqProvider({
groqToken: 'test-token'
});
expect(typeof groqProvider.research).toEqual('function');
let errorCaught = false;
try {
await groqProvider.research({ query: 'test' });
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Research Stubs: Ollama provider should throw not supported error', async () => {
const ollamaProvider = new smartai.OllamaProvider({});
expect(typeof ollamaProvider.research).toEqual('function');
let errorCaught = false;
try {
await ollamaProvider.research({ query: 'test' });
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Research Stubs: xAI provider should throw not supported error', async () => {
const xaiProvider = new smartai.XAIProvider({
xaiToken: 'test-token'
});
expect(typeof xaiProvider.research).toEqual('function');
let errorCaught = false;
try {
await xaiProvider.research({ query: 'test' });
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Research Stubs: Exo provider should throw not supported error', async () => {
const exoProvider = new smartai.ExoProvider({});
expect(typeof exoProvider.research).toEqual('function');
let errorCaught = false;
try {
await exoProvider.research({ query: 'test' });
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
export default tap.start();

View File

@@ -1,65 +0,0 @@
import { tap, expect } from '@push.rocks/tapbundle';
import * as smartai from '../ts/index.js';
// Test the research capabilities
tap.test('OpenAI research method should exist', async () => {
const openaiProvider = new smartai.OpenAiProvider({
openaiToken: 'test-token'
});
// Check that the research method exists
expect(typeof openaiProvider.research).toEqual('function');
});
tap.test('Anthropic research method should exist', async () => {
const anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: 'test-token'
});
// Check that the research method exists
expect(typeof anthropicProvider.research).toEqual('function');
});
tap.test('Research interfaces should be exported', async () => {
// Check that the types are available (they won't be at runtime but TypeScript will check)
const testResearchOptions: smartai.ResearchOptions = {
query: 'test query',
searchDepth: 'basic'
};
expect(testResearchOptions).toBeInstanceOf(Object);
expect(testResearchOptions.query).toEqual('test query');
});
tap.test('Perplexity provider should have research method', async () => {
const perplexityProvider = new smartai.PerplexityProvider({
perplexityToken: 'test-token'
});
// For Perplexity, we actually implemented it, so let's just check it exists
expect(typeof perplexityProvider.research).toEqual('function');
});
tap.test('Other providers should have research stubs', async () => {
const groqProvider = new smartai.GroqProvider({
groqToken: 'test-token'
});
const ollamaProvider = new smartai.OllamaProvider({});
// Check that the research method exists and throws error
expect(typeof groqProvider.research).toEqual('function');
expect(typeof ollamaProvider.research).toEqual('function');
// Test that they throw errors when called
let errorCaught = false;
try {
await groqProvider.research({ query: 'test' });
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartai',
version: '0.6.0',
version: '0.6.1',
description: 'SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.'
}

View File

@@ -253,7 +253,7 @@ export class AnthropicProvider extends MultiModalModel {
// Build the tool configuration for web search
const tools = this.options.enableWebSearch ? [
{
type: 'computer_20241022' as const,
type: 'web_search_20250305' as const,
name: 'web_search',
description: 'Search the web for current information',
input_schema: {