smartai/ts/provider.openai.ts

import * as plugins from './plugins.js';
import * as paths from './paths.js';

// Custom type definition for chat completion messages
export type TChatCompletionRequestMessage = { 
  role: "system" | "user" | "assistant"; 
  content: string; 
};

import { MultiModalModel } from './abstract.classes.multimodal.js';

export interface IOpenaiProviderOptions {
  openaiToken: string;
  chatModel?: string;
  audioModel?: string;
  visionModel?: string;
  // Optionally add more model options (e.g., documentModel) if needed.
}

export class OpenAiProvider extends MultiModalModel {
  private options: IOpenaiProviderOptions;
  public openAiApiClient: plugins.openai.default;
  public smartpdfInstance: plugins.smartpdf.SmartPdf;

  constructor(optionsArg: IOpenaiProviderOptions) {
    super();
    this.options = optionsArg;
  }

  public async start() {
    this.openAiApiClient = new plugins.openai.default({
      apiKey: this.options.openaiToken,
      dangerouslyAllowBrowser: true,
    });
    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
  }

  public async stop() {}

  public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
    // Create a TextDecoder to handle incoming chunks
    const decoder = new TextDecoder();
    let buffer = '';
    let currentMessage: { 
      role: "function" | "user" | "system" | "assistant" | "tool" | "developer"; 
      content: string; 
    } | null = null;

    // Create a TransformStream to process the input
    const transform = new TransformStream<Uint8Array, string>({
      transform: async (chunk, controller) => {
        buffer += decoder.decode(chunk, { stream: true });

        // Try to parse complete JSON messages from the buffer
        while (true) {
          const newlineIndex = buffer.indexOf('\n');
          if (newlineIndex === -1) break;

          const line = buffer.slice(0, newlineIndex);
          buffer = buffer.slice(newlineIndex + 1);

          if (line.trim()) {
            try {
              const message = JSON.parse(line);
              currentMessage = {
                role: (message.role || 'user') as "function" | "user" | "system" | "assistant" | "tool" | "developer",
                content: message.content || '',
              };
            } catch (e) {
              console.error('Failed to parse message:', e);
            }
          }
        }

        // If we have a complete message, send it to OpenAI
        if (currentMessage) {
          const messageToSend = { role: "user" as const, content: currentMessage.content };
          const chatModel = this.options.chatModel ?? 'o3-mini';
          const requestParams: any = {
            model: chatModel,
            messages: [messageToSend],
            stream: true,
          };
          // Temperature is omitted since the model does not support it.
          const stream = await this.openAiApiClient.chat.completions.create(requestParams);
          // Explicitly cast the stream as an async iterable to satisfy TypeScript.
          const streamAsyncIterable = stream as unknown as AsyncIterableIterator<any>;
          // Process each chunk from OpenAI
          for await (const chunk of streamAsyncIterable) {
            const content = chunk.choices[0]?.delta?.content;
            if (content) {
              controller.enqueue(content);
            }
          }
          currentMessage = null;
        }
      },

      flush(controller) {
        if (buffer) {
          try {
            const message = JSON.parse(buffer);
            controller.enqueue(message.content || '');
          } catch (e) {
            console.error('Failed to parse remaining buffer:', e);
          }
        }
      }
    });

    // Connect the input to our transform stream
    return input.pipeThrough(transform);
  }

  // Implementing the synchronous chat interaction
  public async chat(optionsArg: {
    systemMessage: string;
    userMessage: string;
    messageHistory: {
      role: 'assistant' | 'user';
      content: string;
    }[];
  }) {
    const chatModel = this.options.chatModel ?? 'o3-mini';
    const requestParams: any = {
      model: chatModel,
      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        { role: 'user', content: optionsArg.userMessage },
      ],
    };
    // Temperature parameter removed to avoid unsupported error.
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return {
      role: result.choices[0].message.role as 'assistant',
      message: result.choices[0].message.content,
    };
  }

  public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
    const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
    const result = await this.openAiApiClient.audio.speech.create({
      model: this.options.audioModel ?? 'o3-mini',
      input: optionsArg.message,
      voice: 'nova',
      response_format: 'mp3',
      speed: 1,
    });
    const stream = result.body;
    done.resolve(stream);
    return done.promise;
  }

  public async document(optionsArg: {
    systemMessage: string;
    userMessage: string;
    pdfDocuments: Uint8Array[];
    messageHistory: {
      role: 'assistant' | 'user';
      content: any;
    }[];
  }) {
    let pdfDocumentImageBytesArray: Uint8Array[] = [];

    // Convert each PDF into one or more image byte arrays.
    const smartpdfInstance = new plugins.smartpdf.SmartPdf();
    await smartpdfInstance.start();
    for (const pdfDocument of optionsArg.pdfDocuments) {
      const documentImageArray = await smartpdfInstance.convertPDFToPngBytes(pdfDocument);
      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
    }
    await smartpdfInstance.stop();

    console.log(`image smartfile array`);
    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));

    // Filter out any empty buffers to avoid sending invalid image URLs.
    const validImageBytesArray = pdfDocumentImageBytesArray.filter(imageBytes => imageBytes && imageBytes.length > 0);
    const imageAttachments = validImageBytesArray.map(imageBytes => ({
      type: 'image_url',
      image_url: {
        url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
      },
    }));

    const chatModel = this.options.chatModel ?? 'gpt-4o';
    const requestParams: any = {
      model: chatModel,
      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        {
          role: 'user',
          content: [
            { type: 'text', text: optionsArg.userMessage },
            ...imageAttachments,
          ],
        },
      ],
    };
    // Temperature parameter removed.
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return {
      message: result.choices[0].message,
    };
  }

  public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
    const visionModel = this.options.visionModel ?? 'gpt-4o';
    const requestParams: any = {
      model: visionModel,
      messages: [
        {
          role: 'user',
          content: [
            { type: 'text', text: optionsArg.prompt },
            {
              type: 'image_url',
              image_url: {
                url: `data:image/jpeg;base64,${optionsArg.image.toString('base64')}`
              }
            }
          ]
        }
      ],
      max_tokens: 300
    };
    const result = await this.openAiApiClient.chat.completions.create(requestParams);
    return result.choices[0].message.content || '';
  }
}