feat(ollama): add support for base64-encoded images in chat messages and forward them to the Ollama provider

ts/provider.ollama.ts

@@ -43,6 +43,7 @@ export interface IOllamaChatOptions extends ChatOptions {
   options?: IOllamaModelOptions;  // Per-request model options
   timeout?: number;               // Per-request timeout in ms
   model?: string;                 // Per-request model override
+  // images is inherited from ChatOptions
 }
 
 /**
@@ -203,10 +204,30 @@ export class OllamaProvider extends MultiModalModel {
   // Implementing the synchronous chat interaction
   public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
     // Format messages for Ollama
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
-      { role: 'user', content: optionsArg.userMessage }
+      ...historyMessages,
+      userMessage,
     ];
 
     // Make API call to Ollama with defaultOptions and timeout
@@ -243,12 +264,13 @@ export class OllamaProvider extends MultiModalModel {
   public async chatStreaming(optionsArg: StreamingChatOptions): Promise<ChatResponse> {
     const onToken = optionsArg.onToken;
 
-    // Use existing collectStreamResponse with callback
+    // Use existing collectStreamResponse with callback, including images
     const response = await this.collectStreamResponse(
       {
         systemMessage: optionsArg.systemMessage,
         userMessage: optionsArg.userMessage,
         messageHistory: optionsArg.messageHistory,
+        images: optionsArg.images,
       },
       (chunk) => {
         if (onToken) {
@@ -274,10 +296,31 @@ export class OllamaProvider extends MultiModalModel {
     const timeout = optionsArg.timeout || this.defaultTimeout;
     const modelOptions = { ...this.defaultOptions, ...optionsArg.options };
 
+    // Format history messages with optional images
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
-      { role: 'user', content: optionsArg.userMessage }
+      ...historyMessages,
+      userMessage,
     ];
 
     const response = await fetch(`${this.baseUrl}/api/chat`, {
@@ -367,10 +410,31 @@ export class OllamaProvider extends MultiModalModel {
     const timeout = optionsArg.timeout || this.defaultTimeout;
     const modelOptions = { ...this.defaultOptions, ...optionsArg.options };
 
+    // Format history messages with optional images
+    const historyMessages = optionsArg.messageHistory.map((msg) => {
+      const formatted: { role: string; content: string; images?: string[] } = {
+        role: msg.role,
+        content: msg.content,
+      };
+      if (msg.images && msg.images.length > 0) {
+        formatted.images = msg.images;
+      }
+      return formatted;
+    });
+
+    // Build user message with optional images
+    const userMessage: { role: string; content: string; images?: string[] } = {
+      role: 'user',
+      content: optionsArg.userMessage,
+    };
+    if (optionsArg.images && optionsArg.images.length > 0) {
+      userMessage.images = optionsArg.images;
+    }
+
     const messages = [
       { role: 'system', content: optionsArg.systemMessage },
-      ...optionsArg.messageHistory,
-      { role: 'user', content: optionsArg.userMessage }
+      ...historyMessages,
+      userMessage,
     ];
 
     const response = await fetch(`${this.baseUrl}/api/chat`, {