feat(ollama): add support for base64-encoded images in chat messages and forward them to the Ollama provider

changelog.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 2026-01-20 - 0.12.0 - feat(ollama)
+add support for base64-encoded images in chat messages and forward them to the Ollama provider
+
+- Add optional images?: string[] to ChatMessage and ChatOptions interfaces (multimodal/vision support)
+- Propagate images from messageHistory and ChatOptions to the Ollama API payload in chat, chatStreaming, and streaming handlers
+- Changes are non-breaking: images are optional and existing behavior is preserved when absent
+
 ## 2026-01-20 - 0.11.0 - feat(ollama)
 support defaultOptions and defaultTimeout for ollama provider
 

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartai',
-  version: '0.11.0',
+  version: '0.12.0',
   description: 'SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.'
 }

ts/abstract.classes.multimodal.ts

@@ -6,6 +6,8 @@ import * as plugins from './plugins.js';
 export interface ChatMessage {
   role: 'assistant' | 'user' | 'system';
   content: string;
+  /** Base64-encoded images for vision-capable models */
+  images?: string[];
 }
 
 /**
@@ -15,6 +17,8 @@ export interface ChatOptions {
   systemMessage: string;
   userMessage: string;
   messageHistory: ChatMessage[];
+  /** Base64-encoded images for the current message (vision-capable models) */
+  images?: string[];
 }
 
 /**

ts/provider.ollama.ts

@@ -43,6 +43,7 @@ export interface IOllamaChatOptions extends ChatOptions {
   options?: IOllamaModelOptions;  // Per-request model options
   timeout?: number;               // Per-request timeout in ms
   model?: string;                 // Per-request model override
+  // images is inherited from ChatOptions
 }
 
 /**
@@ -203,10 +204,30 @@ export class OllamaProvider extends MultiModalModel {
   // Implementing the synchronous chat interaction
   public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
     // Format messages for Ollama
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
+      ...historyMessages,
-      { role: 'user', content: optionsArg.userMessage }
+      userMessage,
     ];
 
     // Make API call to Ollama with defaultOptions and timeout
@@ -243,12 +264,13 @@ export class OllamaProvider extends MultiModalModel {
   public async chatStreaming(optionsArg: StreamingChatOptions): Promise<ChatResponse> {
     const onToken = optionsArg.onToken;
 
-    // Use existing collectStreamResponse with callback
+    // Use existing collectStreamResponse with callback, including images
     const response = await this.collectStreamResponse(
       {
         systemMessage: optionsArg.systemMessage,
         userMessage: optionsArg.userMessage,
         messageHistory: optionsArg.messageHistory,
+        images: optionsArg.images,
       },
       (chunk) => {
         if (onToken) {
@@ -274,10 +296,31 @@ export class OllamaProvider extends MultiModalModel {
     const timeout = optionsArg.timeout || this.defaultTimeout;
     const modelOptions = { ...this.defaultOptions, ...optionsArg.options };
 
+    // Format history messages with optional images
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
+      ...historyMessages,
-      { role: 'user', content: optionsArg.userMessage }
+      userMessage,
     ];
 
     const response = await fetch(`${this.baseUrl}/api/chat`, {
@@ -367,10 +410,31 @@ export class OllamaProvider extends MultiModalModel {
     const timeout = optionsArg.timeout || this.defaultTimeout;
     const modelOptions = { ...this.defaultOptions, ...optionsArg.options };
 
+    // Format history messages with optional images
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
+      ...historyMessages,
-      { role: 'user', content: optionsArg.userMessage }
+      userMessage,
     ];
 
     const response = await fetch(`${this.baseUrl}/api/chat`, {