smartai/ts/provider.openai.ts

import * as plugins from './plugins.js';
import * as paths from './paths.js';
import { Readable } from 'stream';

// Custom type definition for chat completion messages
export type TChatCompletionRequestMessage = { 
  role: "system" | "user" | "assistant"; 
  content: string; 
};

import { MultiModalModel } from './abstract.classes.multimodal.js';
import type { ResearchOptions, ResearchResponse } from './abstract.classes.multimodal.js';

export interface IOpenaiProviderOptions {
  openaiToken: string;
  chatModel?: string;
  audioModel?: string;
  visionModel?: string;
  researchModel?: string;
  enableWebSearch?: boolean;
}

export class OpenAiProvider extends MultiModalModel {
  private options: IOpenaiProviderOptions;
  public openAiApiClient: plugins.openai.default;

  constructor(optionsArg: IOpenaiProviderOptions) {
    super();
    this.options = optionsArg;
  }

  public async start() {
    await super.start();
    this.openAiApiClient = new plugins.openai.default({
      apiKey: this.options.openaiToken,
      dangerouslyAllowBrowser: true,
    });
  }

  public async stop() {
    await super.stop();
  }

  public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
    // Create a TextDecoder to handle incoming chunks
    const decoder = new TextDecoder();
    let buffer = '';
    let currentMessage: { 
      role: "function" | "user" | "system" | "assistant" | "tool" | "developer"; 
      content: string; 
    } | null = null;

    // Create a TransformStream to process the input
    const transform = new TransformStream<Uint8Array, string>({
      transform: async (chunk, controller) => {
        buffer += decoder.decode(chunk, { stream: true });

        // Try to parse complete JSON messages from the buffer
        while (true) {
          const newlineIndex = buffer.indexOf('\n');
          if (newlineIndex === -1) break;

          const line = buffer.slice(0, newlineIndex);
          buffer = buffer.slice(newlineIndex + 1);

          if (line.trim()) {
            try {
              const message = JSON.parse(line);
              currentMessage = {
                role: (message.role || 'user') as "function" | "user" | "system" | "assistant" | "tool" | "developer",
                content: message.content || '',
              };
            } catch (e) {
              console.error('Failed to parse message:', e);
            }
          }
        }

        // If we have a complete message, send it to OpenAI
        if (currentMessage) {
          const messageToSend = { role: "user" as const, content: currentMessage.content };
          const chatModel = this.options.chatModel ?? 'gpt-5-mini';
          const requestParams: any = {
            model: chatModel,
            messages: [messageToSend],
            stream: true,
          };
          // Temperature is omitted since the model does not support it.
          const stream = await this.openAiApiClient.chat.completions.create(requestParams);
          // Explicitly cast the stream as an async iterable to satisfy TypeScript.
          const streamAsyncIterable = stream as unknown as AsyncIterableIterator<any>;
          // Process each chunk from OpenAI
          for await (const chunk of streamAsyncIterable) {
            const content = chunk.choices[0]?.delta?.content;
            if (content) {
              controller.enqueue(content);
            }
          }
          currentMessage = null;
        }
      },

      flush(controller) {
        if (buffer) {
          try {
            const message = JSON.parse(buffer);
            controller.enqueue(message.content || '');
          } catch (e) {
            console.error('Failed to parse remaining buffer:', e);
          }
        }
      }
    });

    // Connect the input to our transform stream
    return input.pipeThrough(transform);
  }

  // Implementing the synchronous chat interaction
  public async chat(optionsArg: {
    systemMessage: string;
    userMessage: string;
    messageHistory: {
      role: 'assistant' | 'user';
      content: string;
    }[];
  }) {
    const chatModel = this.options.chatModel ?? 'gpt-5-mini';
    const requestParams: any = {
      model: chatModel,
      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        { role: 'user', content: optionsArg.userMessage },
      ],
    };
    // Temperature parameter removed to avoid unsupported error.
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return {
      role: result.choices[0].message.role as 'assistant',
      message: result.choices[0].message.content,
    };
  }

  public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
    const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
    const result = await this.openAiApiClient.audio.speech.create({
      model: this.options.audioModel ?? 'tts-1-hd',
      input: optionsArg.message,
      voice: 'nova',
      response_format: 'mp3',
      speed: 1,
    });
    const stream = result.body;
    const nodeStream = Readable.fromWeb(stream as any);
    done.resolve(nodeStream);
    return done.promise;
  }

  public async document(optionsArg: {
    systemMessage: string;
    userMessage: string;
    pdfDocuments: Uint8Array[];
    messageHistory: {
      role: 'assistant' | 'user';
      content: any;
    }[];
  }) {
    let pdfDocumentImageBytesArray: Uint8Array[] = [];

    // Convert each PDF into one or more image byte arrays.
    for (const pdfDocument of optionsArg.pdfDocuments) {
      const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
    }

    console.log(`image smartfile array`);
    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));

    // Filter out any empty buffers to avoid sending invalid image URLs.
    const validImageBytesArray = pdfDocumentImageBytesArray.filter(imageBytes => imageBytes && imageBytes.length > 0);
    const imageAttachments = validImageBytesArray.map(imageBytes => ({
      type: 'image_url',
      image_url: {
        url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
      },
    }));

    const chatModel = this.options.chatModel ?? 'gpt-5-mini';
    const requestParams: any = {
      model: chatModel,
      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        {
          role: 'user',
          content: [
            { type: 'text', text: optionsArg.userMessage },
            ...imageAttachments,
          ],
        },
      ],
    };
    // Temperature parameter removed.
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return {
      message: result.choices[0].message,
    };
  }

  public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
    const visionModel = this.options.visionModel ?? '04-mini';
    const requestParams: any = {
      model: visionModel,
      messages: [
        {
          role: 'user',
          content: [
            { type: 'text', text: optionsArg.prompt },
            {
              type: 'image_url',
              image_url: {
                url: `data:image/jpeg;base64,${optionsArg.image.toString('base64')}`
              }
            }
          ]
        }
      ],
      max_tokens: 300
    };
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return result.choices[0].message.content || '';
  }

  public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
    // Determine which model to use based on search depth
    let model: string;
    if (optionsArg.searchDepth === 'deep') {
      model = this.options.researchModel || 'o4-mini-deep-research-2025-06-26';
    } else {
      model = this.options.chatModel || 'gpt-5-mini';
    }

    // Prepare the request parameters
    const requestParams: any = {
      model,
      messages: [
        {
          role: 'system',
          content: 'You are a research assistant. Provide comprehensive answers with citations and sources when available.'
        },
        {
          role: 'user',
          content: optionsArg.query
        }
      ],
      temperature: 0.7
    };

    // Add web search tools if requested
    if (optionsArg.includeWebSearch || optionsArg.searchDepth === 'deep') {
      requestParams.tools = [
        {
          type: 'function',
          function: {
            name: 'web_search',
            description: 'Search the web for information',
            parameters: {
              type: 'object',
              properties: {
                query: {
                  type: 'string',
                  description: 'The search query'
                }
              },
              required: ['query']
            }
          }
        }
      ];
      requestParams.tool_choice = 'auto';
    }

    // Add background flag for deep research
    if (optionsArg.background && optionsArg.searchDepth === 'deep') {
      requestParams.background = true;
    }

    try {
      // Execute the research request
      const result = await this.openAiApiClient.chat.completions.create(requestParams);

      // Extract the answer
      const answer = result.choices[0].message.content || '';

      // Parse sources from the response (OpenAI often includes URLs in markdown format)
      const sources: Array<{ url: string; title: string; snippet: string }> = [];
      const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
      let match: RegExpExecArray | null;

      while ((match = urlRegex.exec(answer)) !== null) {
        sources.push({
          title: match[1],
          url: match[2],
          snippet: '' // OpenAI doesn't provide snippets in standard responses
        });
      }

      // Extract search queries if tools were used
      const searchQueries: string[] = [];
      if (result.choices[0].message.tool_calls) {
        for (const toolCall of result.choices[0].message.tool_calls) {
          if ('function' in toolCall && toolCall.function.name === 'web_search') {
            try {
              const args = JSON.parse(toolCall.function.arguments);
              if (args.query) {
                searchQueries.push(args.query);
              }
            } catch (e) {
              // Ignore parsing errors
            }
          }
        }
      }

      return {
        answer,
        sources,
        searchQueries: searchQueries.length > 0 ? searchQueries : undefined,
        metadata: {
          model,
          searchDepth: optionsArg.searchDepth || 'basic',
          tokensUsed: result.usage?.total_tokens
        }
      };
    } catch (error) {
      console.error('Research API error:', error);
      throw new Error(`Failed to perform research: ${error.message}`);
    }
  }
}