Compare commits

..

13 Commits

Author SHA1 Message Date
af3b61cf74 0.7.6
Some checks failed
Default (tags) / security (push) Failing after 23s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-09 07:00:15 +00:00
8666876879 fix(provider.elevenlabs): Provide default ElevenLabs TTS voice fallback and add local tool/project configs 2025-10-09 07:00:15 +00:00
b78168307b 0.7.5
Some checks failed
Default (tags) / security (push) Failing after 24s
Default (tags) / test (push) Failing after 15s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-08 22:56:53 +00:00
bbd8770205 fix(provider.elevenlabs): Update ElevenLabs default TTS model to eleven_v3 and add local Claude permissions file 2025-10-08 22:56:53 +00:00
28bb13dc0c update 2025-10-08 22:49:08 +00:00
3a24c2c4bd 0.7.4
Some checks failed
Default (tags) / security (push) Failing after 21s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-03 15:47:15 +00:00
8244ac6eb0 fix(provider.anthropic): Use image/png for embedded PDF images in Anthropic provider and add local Claude settings for development permissions 2025-10-03 15:47:15 +00:00
2791d738d6 0.7.3
Some checks failed
Default (tags) / security (push) Failing after 22s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-03 14:21:25 +00:00
3fbd054985 fix(tests): Add extensive provider/feature tests and local Claude CI permissions 2025-10-03 14:21:25 +00:00
8e8830ef92 0.7.2
Some checks failed
Default (tags) / security (push) Failing after 14s
Default (tags) / test (push) Failing after 14s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-03 13:51:49 +00:00
34931875ad fix(anthropic): Update Anthropic provider branding to Claude Sonnet 4.5 and add local Claude permissions 2025-10-03 13:51:49 +00:00
2672509d3f 0.7.1
Some checks failed
Default (tags) / security (push) Failing after 23s
Default (tags) / test (push) Failing after 13s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2025-10-03 13:49:46 +00:00
ee3a635852 fix(docs): Add README image generation docs and .claude local settings 2025-10-03 13:49:46 +00:00
21 changed files with 2717 additions and 957 deletions

1
.serena/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/cache

67
.serena/project.yml Normal file
View File

@@ -0,0 +1,67 @@
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
# * For C, use cpp
# * For JavaScript, use typescript
# Special requirements:
# * csharp: Requires the presence of a .sln file in the project folder.
language: typescript
# whether to use the project's gitignore file to ignore files
# Added on 2025-04-07
ignore_all_files_in_gitignore: true
# list of additional paths to ignore
# same syntax as gitignore, so you can use * and **
# Was previously called `ignored_dirs`, please update your config if you are using that.
# Added (renamed) on 2025-04-07
ignored_paths: []
# whether the project is in read-only mode
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
# Added on 2025-04-18
read_only: false
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
# Below is the complete list of tools for convenience.
# To make sure you have the latest list of tools, and to view their descriptions,
# execute `uv run scripts/print_tool_overview.py`.
#
# * `activate_project`: Activates a project by name.
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
# * `create_text_file`: Creates/overwrites a file in the project directory.
# * `delete_lines`: Deletes a range of lines within a file.
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
# * `execute_shell_command`: Executes a shell command.
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
# * `initial_instructions`: Gets the initial instructions for the current project.
# Should only be used in settings where the system prompt cannot be set,
# e.g. in clients you have no control over, like Claude Desktop.
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
# * `insert_at_line`: Inserts content at a given line in a file.
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
# * `list_memories`: Lists memories in Serena's project-specific memory store.
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
# * `read_file`: Reads a file within the project directory.
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
# * `remove_project`: Removes a project from the Serena configuration.
# * `replace_lines`: Replaces a range of lines within a file with new content.
# * `replace_symbol_body`: Replaces the full definition of a symbol.
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
# * `search_for_pattern`: Performs a search for a pattern in the project.
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
# * `switch_modes`: Activates modes by providing a list of their names
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
excluded_tools: []
# initial prompt for the project. It will always be given to the LLM upon activating the project
# (contrary to the memories, which are loaded on demand).
initial_prompt: ""
project_name: "smartai"

View File

@@ -1,5 +1,47 @@
# Changelog
## 2025-10-09 - 0.7.6 - fix(provider.elevenlabs)
Provide default ElevenLabs TTS voice fallback and add local tool/project configs
- ElevenLabsProvider: fallback to Samara voice id ('19STyYD15bswVz51nqLf') when no voiceId or defaultVoiceId is provided — avoids throwing an error on TTS calls.
- ElevenLabsProvider: continue to use 'eleven_v3' as the default model for TTS.
- Add .claude/settings.local.json with expanded allowed permissions for local tooling and web search.
- Add .serena/project.yml and .serena/.gitignore to include Serena project configuration and ignore cache.
## 2025-10-08 - 0.7.5 - fix(provider.elevenlabs)
Update ElevenLabs default TTS model to eleven_v3 and add local Claude permissions file
- Changed default ElevenLabs modelId from 'eleven_multilingual_v2' to 'eleven_v3' in ts/provider.elevenlabs.ts to use the newer/default TTS model.
- Added .claude/settings.local.json with a permissions allow-list for local Claude tooling and CI tasks.
## 2025-10-03 - 0.7.4 - fix(provider.anthropic)
Use image/png for embedded PDF images in Anthropic provider and add local Claude settings for development permissions
- AnthropicProvider: change media_type from 'image/jpeg' to 'image/png' when embedding images extracted from PDFs to ensure correct format in Anthropic requests.
- Add .claude/settings.local.json with development/testing permissions for local Claude usage (shell commands, webfetch, websearch, test/run tasks).
## 2025-10-03 - 0.7.3 - fix(tests)
Add extensive provider/feature tests and local Claude CI permissions
- Add many focused test files covering providers and features: OpenAI, Anthropic, Perplexity, Groq, Ollama, Exo, XAI (chat, audio, vision, document, research, image generation, stubs, interfaces, basic)
- Introduce .claude/settings.local.json to declare allowed permissions for local Claude/CI actions
- Replace older aggregated test files with modular per-feature tests (removed legacy combined tests and split into smaller suites)
- No changes to library runtime code — this change adds tests and CI/local agent configuration only
## 2025-10-03 - 0.7.2 - fix(anthropic)
Update Anthropic provider branding to Claude Sonnet 4.5 and add local Claude permissions
- Docs: Replace 'Claude 3 Opus' with 'Claude Sonnet 4.5' in README provider capabilities matrix.
- Config: Add .claude/settings.local.json to define local Claude permissions for tests and development commands.
## 2025-10-03 - 0.7.1 - fix(docs)
Add README image generation docs and .claude local settings
- Add .claude/settings.local.json with permission allow-list for local assistant tooling and web search
- Update README provider capabilities table to include an Images column and reference gpt-image-1
- Add Image Generation & Editing section with examples, options, and gpt-image-1 advantages
- Mark image generation support as implemented in the roadmap and remove duplicate entry
## 2025-10-03 - 0.7.0 - feat(providers)
Add research API and image generation/editing support; extend providers and tests

View File

@@ -1,6 +1,6 @@
{
"name": "@push.rocks/smartai",
"version": "0.7.0",
"version": "0.7.6",
"private": false,
"description": "SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.",
"main": "dist_ts/index.js",
@@ -15,22 +15,23 @@
"buildDocs": "(tsdoc)"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.6.4",
"@git.zone/tsbuild": "^2.6.8",
"@git.zone/tsbundle": "^2.5.1",
"@git.zone/tsrun": "^1.3.3",
"@git.zone/tstest": "^2.3.2",
"@push.rocks/qenv": "^6.1.0",
"@git.zone/tstest": "^2.3.8",
"@push.rocks/qenv": "^6.1.3",
"@push.rocks/tapbundle": "^6.0.3",
"@types/node": "^22.15.17"
"@types/node": "^22.15.17",
"typescript": "^5.9.3"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.59.0",
"@anthropic-ai/sdk": "^0.65.0",
"@push.rocks/smartarray": "^1.1.0",
"@push.rocks/smartfile": "^11.2.5",
"@push.rocks/smartfile": "^11.2.7",
"@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpdf": "^4.1.1",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^4.2.1",
"@push.rocks/smartrequest": "^4.3.1",
"@push.rocks/webstream": "^1.0.10",
"openai": "^5.12.2"
},

2602
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

128
readme.md
View File

@@ -5,7 +5,7 @@
[![TypeScript](https://img.shields.io/badge/TypeScript-5.x-blue.svg)](https://www.typescriptlang.org/)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
SmartAI unifies the world's leading AI providers - OpenAI, Anthropic, Perplexity, Ollama, Groq, XAI, and Exo - under a single, elegant TypeScript interface. Build AI applications at lightning speed without vendor lock-in.
SmartAI unifies the world's leading AI providers - OpenAI, Anthropic, Perplexity, Ollama, Groq, XAI, Exo, and ElevenLabs - under a single, elegant TypeScript interface. Build AI applications at lightning speed without vendor lock-in.
## 🎯 Why SmartAI?
@@ -28,7 +28,11 @@ import { SmartAi } from '@push.rocks/smartai';
// Initialize with your favorite providers
const ai = new SmartAi({
openaiToken: 'sk-...',
anthropicToken: 'sk-ant-...'
anthropicToken: 'sk-ant-...',
elevenlabsToken: 'sk-...',
elevenlabs: {
defaultVoiceId: '19STyYD15bswVz51nqLf' // Optional: Samara voice
}
});
await ai.start();
@@ -45,15 +49,16 @@ const response = await ai.openaiProvider.chat({
Choose the right provider for your use case:
| Provider | Chat | Streaming | TTS | Vision | Documents | Research | Highlights |
|----------|:----:|:---------:|:---:|:------:|:---------:|:--------:|------------|
| **OpenAI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | • GPT-4, DALL-E 3<br>• Industry standard<br>• Deep research API |
| **Anthropic** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | • Claude 3 Opus<br>• Superior reasoning<br>• Web search API |
| **Ollama** | | ✅ | ❌ | | | ❌ | • 100% local<br>Privacy-first<br>• No API costs |
| **XAI** | ✅ | ✅ | ❌ | | ✅ | ❌ | • Grok models<br>• Real-time data<br>• Uncensored |
| **Perplexity** | ✅ | ✅ | ❌ | ❌ | ❌ | | • Web-aware<br>• Research-focused<br>• Sonar Pro models |
| **Groq** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | • 10x faster<br>LPU inference<br>• Low latency |
| **Exo** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | • Distributed<br>• P2P compute<br>• Decentralized |
| Provider | Chat | Streaming | TTS | Vision | Documents | Research | Images | Highlights |
|----------|:----:|:---------:|:---:|:------:|:---------:|:--------:|:------:|------------|
| **OpenAI** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | • gpt-image-1<br>• DALL-E 3<br>• Deep research API |
| **Anthropic** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | • Claude Sonnet 4.5<br>• Superior reasoning<br>• Web search API |
| **ElevenLabs** | ❌ | ❌ | ✅ | ❌ | | | ❌ | • Premium TTS<br>70+ languages<br>• Natural voices |
| **Ollama** | ✅ | ✅ | ❌ | | ✅ | ❌ | ❌ | • 100% local<br>• Privacy-first<br>• No API costs |
| **XAI** | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | | • Grok models<br>• Real-time data<br>• Uncensored |
| **Perplexity** | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | • Web-aware<br>Research-focused<br>• Sonar Pro models |
| **Groq** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | • 10x faster<br>• LPU inference<br>• Low latency |
| **Exo** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | • Distributed<br>• P2P compute<br>• Decentralized |
## 🎮 Core Features
@@ -105,13 +110,27 @@ while (true) {
### 🎙️ Text-to-Speech
Generate natural voices with OpenAI:
Generate natural voices with OpenAI or ElevenLabs:
```typescript
// OpenAI TTS
const audioStream = await ai.openaiProvider.audio({
message: 'Welcome to the future of AI development!'
});
// ElevenLabs TTS - Premium quality, natural voices (uses v3 by default)
const elevenLabsAudio = await ai.elevenlabsProvider.audio({
message: 'Experience the most lifelike text to speech technology.',
voiceId: '19STyYD15bswVz51nqLf', // Optional: Samara voice
modelId: 'eleven_v3', // Optional: defaults to eleven_v3 (70+ languages, most expressive)
voiceSettings: { // Optional: fine-tune voice characteristics
stability: 0.5, // 0-1: Speech consistency
similarity_boost: 0.8, // 0-1: Voice similarity to original
style: 0.0, // 0-1: Expressiveness (higher = more expressive)
use_speaker_boost: true // Enhanced clarity
}
});
// Stream directly to speakers
audioStream.pipe(speakerOutput);
@@ -216,6 +235,87 @@ const perplexityResearch = await ai.perplexityProvider.research({
- **Anthropic**: Web Search API with domain filtering
- **Perplexity**: Sonar and Sonar Pro models with built-in citations
### 🎨 Image Generation & Editing
Generate and edit images with OpenAI's cutting-edge models:
```typescript
// Basic image generation with gpt-image-1
const image = await ai.openaiProvider.imageGenerate({
prompt: 'A futuristic robot assistant in a modern office, digital art',
model: 'gpt-image-1',
quality: 'high',
size: '1024x1024'
});
// Save the generated image
const imageBuffer = Buffer.from(image.images[0].b64_json!, 'base64');
fs.writeFileSync('robot.png', imageBuffer);
// Advanced: Transparent background with custom format
const logo = await ai.openaiProvider.imageGenerate({
prompt: 'Minimalist mountain peak logo, geometric design',
model: 'gpt-image-1',
quality: 'high',
size: '1024x1024',
background: 'transparent',
outputFormat: 'png'
});
// WebP with compression for web use
const webImage = await ai.openaiProvider.imageGenerate({
prompt: 'Product showcase: sleek smartphone on marble surface',
model: 'gpt-image-1',
quality: 'high',
size: '1536x1024',
outputFormat: 'webp',
outputCompression: 85
});
// Superior text rendering (gpt-image-1's strength)
const signage = await ai.openaiProvider.imageGenerate({
prompt: 'Vintage cafe sign saying "COFFEE & CODE" in hand-lettered typography',
model: 'gpt-image-1',
quality: 'high',
size: '1024x1024'
});
// Generate multiple variations at once
const variations = await ai.openaiProvider.imageGenerate({
prompt: 'Abstract geometric pattern, colorful minimalist art',
model: 'gpt-image-1',
n: 3,
quality: 'medium',
size: '1024x1024'
});
// Edit an existing image
const editedImage = await ai.openaiProvider.imageEdit({
image: originalImageBuffer,
prompt: 'Add sunglasses and change the background to a beach sunset',
model: 'gpt-image-1',
quality: 'high'
});
```
**Image Generation Options:**
- `model`: 'gpt-image-1' | 'dall-e-3' | 'dall-e-2'
- `quality`: 'low' | 'medium' | 'high' | 'auto'
- `size`: Multiple aspect ratios up to 4096×4096
- `background`: 'transparent' | 'opaque' | 'auto'
- `outputFormat`: 'png' | 'jpeg' | 'webp'
- `outputCompression`: 0-100 for webp/jpeg
- `moderation`: 'low' | 'auto'
- `n`: Number of images (1-10)
**gpt-image-1 Advantages:**
- Superior text rendering in images
- Up to 4096×4096 resolution
- Transparent background support
- Advanced output formats (WebP with compression)
- Better prompt understanding
- Streaming support for progressive rendering
### 🔄 Persistent Conversations
Maintain context across interactions:
@@ -467,6 +567,7 @@ npm install @push.rocks/smartai
export OPENAI_API_KEY=sk-...
export ANTHROPIC_API_KEY=sk-ant-...
export PERPLEXITY_API_KEY=pplx-...
export ELEVENLABS_API_KEY=sk-...
# ... etc
```
@@ -493,6 +594,7 @@ export PERPLEXITY_API_KEY=pplx-...
| **Complex Reasoning** | Anthropic | Superior logical thinking, safer outputs |
| **Research & Facts** | Perplexity | Web-aware, provides citations |
| **Deep Research** | OpenAI | Deep Research API with comprehensive analysis |
| **Premium TTS** | ElevenLabs | Most natural voices, 70+ languages, superior quality (v3) |
| **Speed Critical** | Groq | 10x faster inference, sub-second responses |
| **Privacy Critical** | Ollama | 100% local, no data leaves your servers |
| **Real-time Data** | XAI | Access to current information |
@@ -501,8 +603,8 @@ export PERPLEXITY_API_KEY=pplx-...
## 📈 Roadmap
- [x] Research & Web Search API
- [x] Image generation support (gpt-image-1, DALL-E 3, DALL-E 2)
- [ ] Streaming function calls
- [ ] Image generation support
- [ ] Voice input processing
- [ ] Fine-tuning integration
- [ ] Embedding support

View File

@@ -1,216 +0,0 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic: should create and start Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
});
tap.test('Anthropic: should create chat response', async () => {
const userMessage = 'What is the capital of France? Answer in one word.';
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant. Be concise.',
userMessage: userMessage,
messageHistory: [],
});
console.log(`Anthropic Chat - User: ${userMessage}`);
console.log(`Anthropic Chat - Response: ${response.message}`);
expect(response.role).toEqual('assistant');
expect(response.message).toBeTruthy();
expect(response.message.toLowerCase()).toInclude('paris');
});
tap.test('Anthropic: should handle message history', async () => {
const messageHistory: smartai.ChatMessage[] = [
{ role: 'user', content: 'My name is Claude Test' },
{ role: 'assistant', content: 'Nice to meet you, Claude Test!' }
];
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant with good memory.',
userMessage: 'What is my name?',
messageHistory: messageHistory,
});
console.log(`Anthropic Memory Test - Response: ${response.message}`);
expect(response.message.toLowerCase()).toInclude('claude test');
});
tap.test('Anthropic: should analyze coffee image with latte art', async () => {
// Test 1: Coffee image from Unsplash by Dani
const imagePath = './test/testimages/coffee-dani/coffee.jpg';
console.log(`Loading coffee image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'Describe this coffee image. What do you see in terms of the cup, foam pattern, and overall composition?'
});
console.log(`Anthropic Vision (Coffee) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
expect(result.toLowerCase()).toInclude('coffee');
// The image has a heart pattern in the latte art
const mentionsLatte = result.toLowerCase().includes('heart') ||
result.toLowerCase().includes('latte') ||
result.toLowerCase().includes('foam');
expect(mentionsLatte).toBeTrue();
});
tap.test('Anthropic: should analyze laptop/workspace image', async () => {
// Test 2: Laptop image from Unsplash by Nicolas Bichon
const imagePath = './test/testimages/laptop-nicolas/laptop.jpg';
console.log(`Loading laptop image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'Describe the technology and workspace setup in this image. What devices and equipment can you see?'
});
console.log(`Anthropic Vision (Laptop) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
// Should mention laptop, computer, keyboard, or desk
const mentionsTech = result.toLowerCase().includes('laptop') ||
result.toLowerCase().includes('computer') ||
result.toLowerCase().includes('keyboard') ||
result.toLowerCase().includes('desk');
expect(mentionsTech).toBeTrue();
});
tap.test('Anthropic: should analyze receipt/document image', async () => {
// Test 3: Receipt image from Unsplash by Annie Spratt
const imagePath = './test/testimages/receipt-annie/receipt.jpg';
console.log(`Loading receipt image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'What type of document is this? Can you identify any text or numbers visible in the image?'
});
console.log(`Anthropic Vision (Receipt) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
// Should mention receipt, document, text, or paper
const mentionsDocument = result.toLowerCase().includes('receipt') ||
result.toLowerCase().includes('document') ||
result.toLowerCase().includes('text') ||
result.toLowerCase().includes('paper');
expect(mentionsDocument).toBeTrue();
});
tap.test('Anthropic: should document a PDF', async () => {
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
const result = await anthropicProvider.document({
systemMessage: 'Classify the document. Only the following answers are allowed: "invoice", "bank account statement", "contract", "test document", "other". The answer should only contain the keyword for machine use.',
userMessage: 'Classify this document.',
messageHistory: [],
pdfDocuments: [Buffer.from(await pdfResponse.arrayBuffer())],
});
console.log(`Anthropic Document - Result:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic: should handle complex document analysis', async () => {
// Test with the demo PDF if it exists
const pdfPath = './.nogit/demo_without_textlayer.pdf';
let pdfBuffer: Uint8Array;
try {
pdfBuffer = await smartfile.fs.toBuffer(pdfPath);
} catch (error) {
// If the file doesn't exist, use the dummy PDF
console.log('Demo PDF not found, using dummy PDF instead');
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
pdfBuffer = Buffer.from(await pdfResponse.arrayBuffer());
}
const result = await anthropicProvider.document({
systemMessage: `
Analyze this document and provide a JSON response with the following structure:
{
"documentType": "string",
"hasText": boolean,
"summary": "string"
}
`,
userMessage: 'Analyze this document.',
messageHistory: [],
pdfDocuments: [pdfBuffer],
});
console.log(`Anthropic Complex Document Analysis:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic: should handle errors gracefully', async () => {
// Test with invalid message (empty)
let errorCaught = false;
try {
await anthropicProvider.chat({
systemMessage: '',
userMessage: '',
messageHistory: [],
});
} catch (error) {
errorCaught = true;
console.log('Expected error caught:', error.message);
}
// Anthropic might handle empty messages, so we don't assert error
console.log(`Error handling test - Error caught: ${errorCaught}`);
});
tap.test('Anthropic: audio should throw not supported error', async () => {
let errorCaught = false;
try {
await anthropicProvider.audio({
message: 'This should fail'
});
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Anthropic: should stop the provider', async () => {
await anthropicProvider.stop();
console.log('Anthropic provider stopped successfully');
});
export default tap.start();

View File

@@ -0,0 +1,54 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let testSmartai: smartai.SmartAi;
tap.test('ElevenLabs Audio: should create a smartai instance with ElevenLabs provider', async () => {
testSmartai = new smartai.SmartAi({
elevenlabsToken: await testQenv.getEnvVarOnDemand('ELEVENLABS_TOKEN'),
elevenlabs: {
defaultVoiceId: '19STyYD15bswVz51nqLf',
},
});
await testSmartai.start();
});
tap.test('ElevenLabs Audio: should create audio response', async () => {
const audioStream = await testSmartai.elevenlabsProvider.audio({
message: 'Welcome to SmartAI, the unified interface for the world\'s leading artificial intelligence providers. SmartAI brings together OpenAI, Anthropic, Perplexity, and ElevenLabs under a single elegant TypeScript API. Whether you need text generation, vision analysis, document processing, or premium text-to-speech capabilities, SmartAI provides a consistent and powerful interface for all your AI needs. Build intelligent applications at lightning speed without vendor lock-in.',
});
const chunks: Uint8Array[] = [];
for await (const chunk of audioStream) {
chunks.push(chunk as Uint8Array);
}
const audioBuffer = Buffer.concat(chunks);
await smartfile.fs.toFs(audioBuffer, './.nogit/testoutput_elevenlabs.mp3');
console.log(`Audio Buffer length: ${audioBuffer.length}`);
expect(audioBuffer.length).toBeGreaterThan(0);
});
tap.test('ElevenLabs Audio: should create audio with custom voice', async () => {
const audioStream = await testSmartai.elevenlabsProvider.audio({
message: 'Testing with a different voice.',
voiceId: 'JBFqnCBsd6RMkjVDRZzb',
});
const chunks: Uint8Array[] = [];
for await (const chunk of audioStream) {
chunks.push(chunk as Uint8Array);
}
const audioBuffer = Buffer.concat(chunks);
await smartfile.fs.toFs(audioBuffer, './.nogit/testoutput_elevenlabs_custom.mp3');
console.log(`Audio Buffer length (custom voice): ${audioBuffer.length}`);
expect(audioBuffer.length).toBeGreaterThan(0);
});
tap.test('ElevenLabs Audio: should stop the smartai instance', async () => {
await testSmartai.stop();
});
export default tap.start();

39
test/test.audio.openai.ts Normal file
View File

@@ -0,0 +1,39 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let testSmartai: smartai.SmartAi;
tap.test('OpenAI Audio: should create a smartai instance with OpenAI provider', async () => {
testSmartai = new smartai.SmartAi({
openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
});
await testSmartai.start();
});
tap.test('OpenAI Audio: should create audio response', async () => {
// Call the audio method with a sample message.
const audioStream = await testSmartai.openaiProvider.audio({
message: 'This is a test of audio generation.',
});
// Read all chunks from the stream.
const chunks: Uint8Array[] = [];
for await (const chunk of audioStream) {
chunks.push(chunk as Uint8Array);
}
const audioBuffer = Buffer.concat(chunks);
await smartfile.fs.toFs(audioBuffer, './.nogit/testoutput.mp3');
console.log(`Audio Buffer length: ${audioBuffer.length}`);
// Assert that the resulting buffer is not empty.
expect(audioBuffer.length).toBeGreaterThan(0);
});
tap.test('OpenAI Audio: should stop the smartai instance', async () => {
await testSmartai.stop();
});
export default tap.start();

36
test/test.audio.stubs.ts Normal file
View File

@@ -0,0 +1,36 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Audio Stubs: should create Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
});
tap.test('Audio Stubs: Anthropic audio should throw not supported error', async () => {
let errorCaught = false;
try {
await anthropicProvider.audio({
message: 'This should fail'
});
} catch (error) {
errorCaught = true;
expect(error.message).toInclude('not yet supported');
}
expect(errorCaught).toBeTrue();
});
tap.test('Audio Stubs: should stop Anthropic provider', async () => {
await anthropicProvider.stop();
});
export default tap.start();

View File

@@ -0,0 +1,72 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic Chat: should create and start Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
});
tap.test('Anthropic Chat: should create chat response', async () => {
const userMessage = 'What is the capital of France? Answer in one word.';
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant. Be concise.',
userMessage: userMessage,
messageHistory: [],
});
console.log(`Anthropic Chat - User: ${userMessage}`);
console.log(`Anthropic Chat - Response: ${response.message}`);
expect(response.role).toEqual('assistant');
expect(response.message).toBeTruthy();
expect(response.message.toLowerCase()).toInclude('paris');
});
tap.test('Anthropic Chat: should handle message history', async () => {
const messageHistory: smartai.ChatMessage[] = [
{ role: 'user', content: 'My name is Claude Test' },
{ role: 'assistant', content: 'Nice to meet you, Claude Test!' }
];
const response = await anthropicProvider.chat({
systemMessage: 'You are a helpful assistant with good memory.',
userMessage: 'What is my name?',
messageHistory: messageHistory,
});
console.log(`Anthropic Memory Test - Response: ${response.message}`);
expect(response.message.toLowerCase()).toInclude('claude test');
});
tap.test('Anthropic Chat: should handle errors gracefully', async () => {
// Test with invalid message (empty)
let errorCaught = false;
try {
await anthropicProvider.chat({
systemMessage: '',
userMessage: '',
messageHistory: [],
});
} catch (error) {
errorCaught = true;
console.log('Expected error caught:', error.message);
}
// Anthropic might handle empty messages, so we don't assert error
console.log(`Error handling test - Error caught: ${errorCaught}`);
});
tap.test('Anthropic Chat: should stop the provider', async () => {
await anthropicProvider.stop();
});
export default tap.start();

34
test/test.chat.openai.ts Normal file
View File

@@ -0,0 +1,34 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let testSmartai: smartai.SmartAi;
tap.test('OpenAI Chat: should create a smartai instance with OpenAI provider', async () => {
testSmartai = new smartai.SmartAi({
openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
});
await testSmartai.start();
});
tap.test('OpenAI Chat: should create chat response', async () => {
const userMessage = 'How are you?';
const response = await testSmartai.openaiProvider.chat({
systemMessage: 'Hello',
userMessage: userMessage,
messageHistory: [],
});
console.log(`userMessage: ${userMessage}`);
console.log(response.message);
expect(response.role).toEqual('assistant');
expect(response.message).toBeTruthy();
});
tap.test('OpenAI Chat: should stop the smartai instance', async () => {
await testSmartai.stop();
});
export default tap.start();

View File

@@ -0,0 +1,78 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartrequest from '@push.rocks/smartrequest';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic Document: should create and start Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
});
tap.test('Anthropic Document: should document a PDF', async () => {
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
const result = await anthropicProvider.document({
systemMessage: 'Classify the document. Only the following answers are allowed: "invoice", "bank account statement", "contract", "test document", "other". The answer should only contain the keyword for machine use.',
userMessage: 'Classify this document.',
messageHistory: [],
pdfDocuments: [Buffer.from(await pdfResponse.arrayBuffer())],
});
console.log(`Anthropic Document - Result:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic Document: should handle complex document analysis', async () => {
// Test with the demo PDF if it exists
const pdfPath = './.nogit/demo_without_textlayer.pdf';
let pdfBuffer: Uint8Array;
try {
pdfBuffer = await smartfile.fs.toBuffer(pdfPath);
} catch (error) {
// If the file doesn't exist, use the dummy PDF
console.log('Demo PDF not found, using dummy PDF instead');
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
.get();
pdfBuffer = Buffer.from(await pdfResponse.arrayBuffer());
}
const result = await anthropicProvider.document({
systemMessage: `
Analyze this document and provide a JSON response with the following structure:
{
"documentType": "string",
"hasText": boolean,
"summary": "string"
}
`,
userMessage: 'Analyze this document.',
messageHistory: [],
pdfDocuments: [pdfBuffer],
});
console.log(`Anthropic Complex Document Analysis:`, result);
expect(result).toBeTruthy();
expect(result.message).toBeTruthy();
});
tap.test('Anthropic Document: should stop the provider', async () => {
await anthropicProvider.stop();
});
export default tap.start();

View File

@@ -9,25 +9,14 @@ import * as smartai from '../ts/index.js';
let testSmartai: smartai.SmartAi;
tap.test('OpenAI: should create a smartai instance with OpenAI provider', async () => {
tap.test('OpenAI Document: should create a smartai instance with OpenAI provider', async () => {
testSmartai = new smartai.SmartAi({
openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
});
await testSmartai.start();
});
tap.test('OpenAI: should create chat response', async () => {
const userMessage = 'How are you?';
const response = await testSmartai.openaiProvider.chat({
systemMessage: 'Hello',
userMessage: userMessage,
messageHistory: [],
});
console.log(`userMessage: ${userMessage}`);
console.log(response.message);
});
tap.test('OpenAI: should document a pdf', async () => {
tap.test('OpenAI Document: should document a pdf', async () => {
const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
const pdfResponse = await smartrequest.SmartRequest.create()
.url(pdfUrl)
@@ -39,9 +28,10 @@ tap.test('OpenAI: should document a pdf', async () => {
pdfDocuments: [Buffer.from(await pdfResponse.arrayBuffer())],
});
console.log(result);
expect(result.message).toBeTruthy();
});
tap.test('OpenAI: should recognize companies in a pdf', async () => {
tap.test('OpenAI Document: should recognize companies in a pdf', async () => {
const pdfBuffer = await smartfile.fs.toBuffer('./.nogit/demo_without_textlayer.pdf');
const result = await testSmartai.openaiProvider.document({
systemMessage: `
@@ -76,27 +66,11 @@ tap.test('OpenAI: should recognize companies in a pdf', async () => {
pdfDocuments: [pdfBuffer],
});
console.log(result);
expect(result.message).toBeTruthy();
});
tap.test('OpenAI: should create audio response', async () => {
// Call the audio method with a sample message.
const audioStream = await testSmartai.openaiProvider.audio({
message: 'This is a test of audio generation.',
});
// Read all chunks from the stream.
const chunks: Uint8Array[] = [];
for await (const chunk of audioStream) {
chunks.push(chunk as Uint8Array);
}
const audioBuffer = Buffer.concat(chunks);
await smartfile.fs.toFs(audioBuffer, './.nogit/testoutput.mp3');
console.log(`Audio Buffer length: ${audioBuffer.length}`);
// Assert that the resulting buffer is not empty.
expect(audioBuffer.length).toBeGreaterThan(0);
});
tap.test('OpenAI: should stop the smartai instance', async () => {
tap.test('OpenAI Document: should stop the smartai instance', async () => {
await testSmartai.stop();
});
export default tap.start();
export default tap.start();

View File

@@ -0,0 +1,95 @@
import { expect, tap } from '@push.rocks/tapbundle';
import * as qenv from '@push.rocks/qenv';
import * as smartfile from '@push.rocks/smartfile';
const testQenv = new qenv.Qenv('./', './.nogit/');
import * as smartai from '../ts/index.js';
let anthropicProvider: smartai.AnthropicProvider;
tap.test('Anthropic Vision: should create and start Anthropic provider', async () => {
anthropicProvider = new smartai.AnthropicProvider({
anthropicToken: await testQenv.getEnvVarOnDemand('ANTHROPIC_TOKEN'),
});
await anthropicProvider.start();
expect(anthropicProvider).toBeInstanceOf(smartai.AnthropicProvider);
});
tap.test('Anthropic Vision: should analyze coffee image with latte art', async () => {
// Test 1: Coffee image from Unsplash by Dani
const imagePath = './test/testimages/coffee-dani/coffee.jpg';
console.log(`Loading coffee image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'Describe this coffee image. What do you see in terms of the cup, foam pattern, and overall composition?'
});
console.log(`Anthropic Vision (Coffee) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
expect(result.toLowerCase()).toInclude('coffee');
// The image has a heart pattern in the latte art
const mentionsLatte = result.toLowerCase().includes('heart') ||
result.toLowerCase().includes('latte') ||
result.toLowerCase().includes('foam');
expect(mentionsLatte).toBeTrue();
});
tap.test('Anthropic Vision: should analyze laptop/workspace image', async () => {
// Test 2: Laptop image from Unsplash by Nicolas Bichon
const imagePath = './test/testimages/laptop-nicolas/laptop.jpg';
console.log(`Loading laptop image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'Describe the technology and workspace setup in this image. What devices and equipment can you see?'
});
console.log(`Anthropic Vision (Laptop) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
// Should mention laptop, computer, keyboard, or desk
const mentionsTech = result.toLowerCase().includes('laptop') ||
result.toLowerCase().includes('computer') ||
result.toLowerCase().includes('keyboard') ||
result.toLowerCase().includes('desk');
expect(mentionsTech).toBeTrue();
});
tap.test('Anthropic Vision: should analyze receipt/document image', async () => {
// Test 3: Receipt image from Unsplash by Annie Spratt
const imagePath = './test/testimages/receipt-annie/receipt.jpg';
console.log(`Loading receipt image from: ${imagePath}`);
const imageBuffer = await smartfile.fs.toBuffer(imagePath);
console.log(`Image loaded, size: ${imageBuffer.length} bytes`);
const result = await anthropicProvider.vision({
image: imageBuffer,
prompt: 'What type of document is this? Can you identify any text or numbers visible in the image?'
});
console.log(`Anthropic Vision (Receipt) - Result: ${result}`);
expect(result).toBeTruthy();
expect(typeof result).toEqual('string');
// Should mention receipt, document, text, or paper
const mentionsDocument = result.toLowerCase().includes('receipt') ||
result.toLowerCase().includes('document') ||
result.toLowerCase().includes('text') ||
result.toLowerCase().includes('paper');
expect(mentionsDocument).toBeTrue();
});
tap.test('Anthropic Vision: should stop the provider', async () => {
await anthropicProvider.stop();
});
export default tap.start();

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartai',
version: '0.7.0',
version: '0.7.6',
description: 'SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.'
}

View File

@@ -96,6 +96,18 @@ export class Conversation {
return conversation;
}
public static async createWithElevenlabs(smartaiRefArg: SmartAi) {
if (!smartaiRefArg.elevenlabsProvider) {
throw new Error('ElevenLabs provider not available');
}
const conversation = new Conversation(smartaiRefArg, {
processFunction: async (input) => {
return '' // TODO implement proper streaming
}
});
return conversation;
}
// INSTANCE
smartaiRef: SmartAi
private systemMessage: string;

View File

@@ -1,6 +1,7 @@
import { Conversation } from './classes.conversation.js';
import * as plugins from './plugins.js';
import { AnthropicProvider } from './provider.anthropic.js';
import { ElevenLabsProvider } from './provider.elevenlabs.js';
import { OllamaProvider } from './provider.ollama.js';
import { OpenAiProvider } from './provider.openai.js';
import { PerplexityProvider } from './provider.perplexity.js';
@@ -15,6 +16,7 @@ export interface ISmartAiOptions {
perplexityToken?: string;
groqToken?: string;
xaiToken?: string;
elevenlabsToken?: string;
exo?: {
baseUrl?: string;
apiKey?: string;
@@ -24,9 +26,13 @@ export interface ISmartAiOptions {
model?: string;
visionModel?: string;
};
elevenlabs?: {
defaultVoiceId?: string;
defaultModelId?: string;
};
}
export type TProvider = 'openai' | 'anthropic' | 'perplexity' | 'ollama' | 'exo' | 'groq' | 'xai';
export type TProvider = 'openai' | 'anthropic' | 'perplexity' | 'ollama' | 'exo' | 'groq' | 'xai' | 'elevenlabs';
export class SmartAi {
public options: ISmartAiOptions;
@@ -38,6 +44,7 @@ export class SmartAi {
public exoProvider: ExoProvider;
public groqProvider: GroqProvider;
public xaiProvider: XAIProvider;
public elevenlabsProvider: ElevenLabsProvider;
constructor(optionsArg: ISmartAiOptions) {
this.options = optionsArg;
@@ -74,6 +81,14 @@ export class SmartAi {
});
await this.xaiProvider.start();
}
if (this.options.elevenlabsToken) {
this.elevenlabsProvider = new ElevenLabsProvider({
elevenlabsToken: this.options.elevenlabsToken,
defaultVoiceId: this.options.elevenlabs?.defaultVoiceId,
defaultModelId: this.options.elevenlabs?.defaultModelId,
});
await this.elevenlabsProvider.start();
}
if (this.options.ollama) {
this.ollamaProvider = new OllamaProvider({
baseUrl: this.options.ollama.baseUrl,
@@ -107,6 +122,9 @@ export class SmartAi {
if (this.xaiProvider) {
await this.xaiProvider.stop();
}
if (this.elevenlabsProvider) {
await this.elevenlabsProvider.stop();
}
if (this.ollamaProvider) {
await this.ollamaProvider.stop();
}
@@ -134,6 +152,8 @@ export class SmartAi {
return Conversation.createWithGroq(this);
case 'xai':
return Conversation.createWithXai(this);
case 'elevenlabs':
return Conversation.createWithElevenlabs(this);
default:
throw new Error('Provider not available');
}

View File

@@ -7,3 +7,4 @@ export * from './provider.groq.js';
export * from './provider.ollama.js';
export * from './provider.xai.js';
export * from './provider.exo.js';
export * from './provider.elevenlabs.js';

View File

@@ -220,7 +220,7 @@ export class AnthropicProvider extends MultiModalModel {
type: 'image',
source: {
type: 'base64',
media_type: 'image/jpeg',
media_type: 'image/png',
data: Buffer.from(imageBytes).toString('base64')
}
});

114
ts/provider.elevenlabs.ts Normal file
View File

@@ -0,0 +1,114 @@
import * as plugins from './plugins.js';
import { MultiModalModel } from './abstract.classes.multimodal.js';
import type {
ChatOptions,
ChatResponse,
ResearchOptions,
ResearchResponse,
ImageGenerateOptions,
ImageEditOptions,
ImageResponse
} from './abstract.classes.multimodal.js';
export interface IElevenLabsProviderOptions {
elevenlabsToken: string;
defaultVoiceId?: string;
defaultModelId?: string;
}
export interface IElevenLabsVoiceSettings {
stability?: number;
similarity_boost?: number;
style?: number;
use_speaker_boost?: boolean;
}
export class ElevenLabsProvider extends MultiModalModel {
private options: IElevenLabsProviderOptions;
private baseUrl: string = 'https://api.elevenlabs.io/v1';
constructor(optionsArg: IElevenLabsProviderOptions) {
super();
this.options = optionsArg;
}
public async start() {
await super.start();
}
public async stop() {
await super.stop();
}
public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
throw new Error('ElevenLabs does not support chat functionality. This provider is specialized for text-to-speech only.');
}
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
throw new Error('ElevenLabs does not support chat streaming functionality. This provider is specialized for text-to-speech only.');
}
public async audio(optionsArg: {
message: string;
voiceId?: string;
modelId?: string;
voiceSettings?: IElevenLabsVoiceSettings;
}): Promise<NodeJS.ReadableStream> {
// Use Samara voice as default fallback
const voiceId = optionsArg.voiceId || this.options.defaultVoiceId || '19STyYD15bswVz51nqLf';
const modelId = optionsArg.modelId || this.options.defaultModelId || 'eleven_v3';
const url = `${this.baseUrl}/text-to-speech/${voiceId}`;
const requestBody: any = {
text: optionsArg.message,
model_id: modelId,
};
if (optionsArg.voiceSettings) {
requestBody.voice_settings = optionsArg.voiceSettings;
}
const response = await plugins.smartrequest.SmartRequest.create()
.url(url)
.header('xi-api-key', this.options.elevenlabsToken)
.json(requestBody)
.autoDrain(false)
.post();
if (!response.ok) {
const errorText = await response.text();
throw new Error(`ElevenLabs API error: ${response.status} ${response.statusText} - ${errorText}`);
}
const nodeStream = response.streamNode();
return nodeStream;
}
public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
throw new Error('ElevenLabs does not support vision functionality. This provider is specialized for text-to-speech only.');
}
public async document(optionsArg: {
systemMessage: string;
userMessage: string;
pdfDocuments: Uint8Array[];
messageHistory: any[];
}): Promise<{ message: any }> {
throw new Error('ElevenLabs does not support document processing. This provider is specialized for text-to-speech only.');
}
public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
throw new Error('ElevenLabs does not support research capabilities. This provider is specialized for text-to-speech only.');
}
public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
throw new Error('ElevenLabs does not support image generation. This provider is specialized for text-to-speech only.');
}
public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
throw new Error('ElevenLabs does not support image editing. This provider is specialized for text-to-speech only.');
}
}