diff --git a/changelog.md b/changelog.md index 5a84353..48e6c7f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,12 @@ # Changelog +## 2026-01-20 - 0.12.0 - feat(ollama) +add support for base64-encoded images in chat messages and forward them to the Ollama provider + +- Add optional images?: string[] to ChatMessage and ChatOptions interfaces (multimodal/vision support) +- Propagate images from messageHistory and ChatOptions to the Ollama API payload in chat, chatStreaming, and streaming handlers +- Changes are non-breaking: images are optional and existing behavior is preserved when absent + ## 2026-01-20 - 0.11.0 - feat(ollama) support defaultOptions and defaultTimeout for ollama provider diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 231bc7f..7e849f0 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartai', - version: '0.11.0', + version: '0.12.0', description: 'SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.' } diff --git a/ts/abstract.classes.multimodal.ts b/ts/abstract.classes.multimodal.ts index eca8091..5189b12 100644 --- a/ts/abstract.classes.multimodal.ts +++ b/ts/abstract.classes.multimodal.ts @@ -6,6 +6,8 @@ import * as plugins from './plugins.js'; export interface ChatMessage { role: 'assistant' | 'user' | 'system'; content: string; + /** Base64-encoded images for vision-capable models */ + images?: string[]; } /** @@ -15,6 +17,8 @@ export interface ChatOptions { systemMessage: string; userMessage: string; messageHistory: ChatMessage[]; + /** Base64-encoded images for the current message (vision-capable models) */ + images?: string[]; } /** diff --git a/ts/provider.ollama.ts b/ts/provider.ollama.ts index 169e286..f78c67a 100644 --- a/ts/provider.ollama.ts +++ b/ts/provider.ollama.ts @@ -43,6 +43,7 @@ export interface IOllamaChatOptions extends ChatOptions { options?: IOllamaModelOptions; // Per-request model options timeout?: number; // Per-request timeout in ms model?: string; // Per-request model override + // images is inherited from ChatOptions } /** @@ -203,10 +204,30 @@ export class OllamaProvider extends MultiModalModel { // Implementing the synchronous chat interaction public async chat(optionsArg: ChatOptions): Promise { // Format messages for Ollama + const historyMessages = optionsArg.messageHistory.map((msg) => { + const formatted: { role: string; content: string; images?: string[] } = { + role: msg.role, + content: msg.content, + }; + if (msg.images && msg.images.length > 0) { + formatted.images = msg.images; + } + return formatted; + }); + + // Build user message with optional images + const userMessage: { role: string; content: string; images?: string[] } = { + role: 'user', + content: optionsArg.userMessage, + }; + if (optionsArg.images && optionsArg.images.length > 0) { + userMessage.images = optionsArg.images; + } + const messages = [ { role: 'system', content: optionsArg.systemMessage }, - ...optionsArg.messageHistory, - { role: 'user', content: optionsArg.userMessage } + ...historyMessages, + userMessage, ]; // Make API call to Ollama with defaultOptions and timeout @@ -243,12 +264,13 @@ export class OllamaProvider extends MultiModalModel { public async chatStreaming(optionsArg: StreamingChatOptions): Promise { const onToken = optionsArg.onToken; - // Use existing collectStreamResponse with callback + // Use existing collectStreamResponse with callback, including images const response = await this.collectStreamResponse( { systemMessage: optionsArg.systemMessage, userMessage: optionsArg.userMessage, messageHistory: optionsArg.messageHistory, + images: optionsArg.images, }, (chunk) => { if (onToken) { @@ -274,10 +296,31 @@ export class OllamaProvider extends MultiModalModel { const timeout = optionsArg.timeout || this.defaultTimeout; const modelOptions = { ...this.defaultOptions, ...optionsArg.options }; + // Format history messages with optional images + const historyMessages = optionsArg.messageHistory.map((msg) => { + const formatted: { role: string; content: string; images?: string[] } = { + role: msg.role, + content: msg.content, + }; + if (msg.images && msg.images.length > 0) { + formatted.images = msg.images; + } + return formatted; + }); + + // Build user message with optional images + const userMessage: { role: string; content: string; images?: string[] } = { + role: 'user', + content: optionsArg.userMessage, + }; + if (optionsArg.images && optionsArg.images.length > 0) { + userMessage.images = optionsArg.images; + } + const messages = [ { role: 'system', content: optionsArg.systemMessage }, - ...optionsArg.messageHistory, - { role: 'user', content: optionsArg.userMessage } + ...historyMessages, + userMessage, ]; const response = await fetch(`${this.baseUrl}/api/chat`, { @@ -367,10 +410,31 @@ export class OllamaProvider extends MultiModalModel { const timeout = optionsArg.timeout || this.defaultTimeout; const modelOptions = { ...this.defaultOptions, ...optionsArg.options }; + // Format history messages with optional images + const historyMessages = optionsArg.messageHistory.map((msg) => { + const formatted: { role: string; content: string; images?: string[] } = { + role: msg.role, + content: msg.content, + }; + if (msg.images && msg.images.length > 0) { + formatted.images = msg.images; + } + return formatted; + }); + + // Build user message with optional images + const userMessage: { role: string; content: string; images?: string[] } = { + role: 'user', + content: optionsArg.userMessage, + }; + if (optionsArg.images && optionsArg.images.length > 0) { + userMessage.images = optionsArg.images; + } + const messages = [ { role: 'system', content: optionsArg.systemMessage }, - ...optionsArg.messageHistory, - { role: 'user', content: optionsArg.userMessage } + ...historyMessages, + userMessage, ]; const response = await fetch(`${this.baseUrl}/api/chat`, {