smartai/ts/abstract.classes.multimodal.ts

import * as plugins from './plugins.js';

/**
 * Message format for chat interactions
 */
export interface ChatMessage {
  role: 'assistant' | 'user' | 'system';
  content: string;
}

/**
 * Options for chat interactions
 */
export interface ChatOptions {
  systemMessage: string;
  userMessage: string;
  messageHistory: ChatMessage[];
}

/**
 * Response format for chat interactions
 */
export interface ChatResponse {
  role: 'assistant';
  message: string;
}

/**
 * Options for research interactions
 */
export interface ResearchOptions {
  query: string;
  searchDepth?: 'basic' | 'advanced' | 'deep';
  maxSources?: number;
  includeWebSearch?: boolean;
  background?: boolean;
}

/**
 * Response format for research interactions
 */
export interface ResearchResponse {
  answer: string;
  sources: Array<{
    url: string;
    title: string;
    snippet: string;
  }>;
  searchQueries?: string[];
  metadata?: any;
}

/**
 * Abstract base class for multi-modal AI models.
 * Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
 */
export abstract class MultiModalModel {
  /**
   * SmartPdf instance for document processing
   * Shared across all methods that need PDF functionality
   */
  protected smartpdfInstance: plugins.smartpdf.SmartPdf;

  /**
   * Initializes the model and any necessary resources
   * Should be called before using any other methods
   */
  public async start(): Promise<void> {
    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
    await this.smartpdfInstance.start();
  }
  
  /**
   * Cleans up any resources used by the model
   * Should be called when the model is no longer needed
   */
  public async stop(): Promise<void> {
    if (this.smartpdfInstance) {
      await this.smartpdfInstance.stop();
    }
  }

  /**
   * Synchronous chat interaction with the model
   * @param optionsArg Options containing system message, user message, and message history
   * @returns Promise resolving to the assistant's response
   */
  public abstract chat(optionsArg: ChatOptions): Promise<ChatResponse>;
  
  /**
   * Streaming interface for chat interactions
   * Allows for real-time responses from the model
   * @param input Stream of user messages
   * @returns Stream of model responses
   */
  public abstract chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>>;

  /**
   * Text-to-speech conversion
   * @param optionsArg Options containing the message to convert to speech
   * @returns Promise resolving to a readable stream of audio data
   * @throws Error if the provider doesn't support audio generation
   */
  public abstract audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream>;

  /**
   * Vision-language processing
   * @param optionsArg Options containing the image and prompt for analysis
   * @returns Promise resolving to the model's description or analysis of the image
   * @throws Error if the provider doesn't support vision tasks
   */
  public abstract vision(optionsArg: { image: Buffer; prompt: string }): Promise<string>;

  /**
   * Document analysis and processing
   * @param optionsArg Options containing system message, user message, PDF documents, and message history
   * @returns Promise resolving to the model's analysis of the documents
   * @throws Error if the provider doesn't support document processing
   */
  public abstract document(optionsArg: {
    systemMessage: string;
    userMessage: string;
    pdfDocuments: Uint8Array[];
    messageHistory: ChatMessage[];
  }): Promise<{ message: any }>;

  /**
   * Research and web search capabilities
   * @param optionsArg Options containing the research query and configuration
   * @returns Promise resolving to the research results with sources
   * @throws Error if the provider doesn't support research capabilities
   */
  public abstract research(optionsArg: ResearchOptions): Promise<ResearchResponse>;
}