smartai/ts/provider.openai.ts

import * as plugins from './plugins.js';
import * as paths from './paths.js';

import { MultiModalModel } from './abstract.classes.multimodal.js';

export interface IOpenaiProviderOptions {
  openaiToken: string;
}

export class OpenAiProvider extends MultiModalModel {
  private options: IOpenaiProviderOptions;
  public openAiApiClient: plugins.openai.default;
  public smartpdfInstance: plugins.smartpdf.SmartPdf;

  constructor(optionsArg: IOpenaiProviderOptions) {
    super();
    this.options = optionsArg;
  }

  public async start() {
    this.openAiApiClient = new plugins.openai.default({
      apiKey: this.options.openaiToken,
      dangerouslyAllowBrowser: true,
    });
    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
  }

  public async stop() {}

  public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
    // TODO: implement for OpenAI

    const returnStream = new ReadableStream();
    return returnStream;
  }

  // Implementing the synchronous chat interaction
  public async chat(optionsArg: {
    systemMessage: string;
    userMessage: string;
    messageHistory: {
      role: 'assistant' | 'user';
      content: string;
    }[];
  }) {
    const result = await this.openAiApiClient.chat.completions.create({
      model: 'gpt-4-turbo-preview',

      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        { role: 'user', content: optionsArg.userMessage },
      ],
    });
    return {
      role: result.choices[0].message.role as 'assistant',
      message: result.choices[0].message.content,
    };
  }

  public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
    const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
    const result = await this.openAiApiClient.audio.speech.create({
      model: 'tts-1-hd',
      input: optionsArg.message,
      voice: 'nova',
      response_format: 'mp3',
      speed: 1,
    });
    const stream = result.body;
    done.resolve(stream);
    return done.promise;
  }

  public async document(optionsArg: {
    systemMessage: string;
    userMessage: string;
    pdfDocuments: Uint8Array[];
    messageHistory: {
      role: 'assistant' | 'user';
      content: any;
    }[];
  }) {
    let pdfDocumentImageBytesArray: Uint8Array[] = [];

    for (const pdfDocument of optionsArg.pdfDocuments) {
      const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
    }

    console.log(`image smartfile array`);
    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));

    const smartfileArray = await plugins.smartarray.map(
      pdfDocumentImageBytesArray,
      async (pdfDocumentImageBytes) => {
        return plugins.smartfile.SmartFile.fromBuffer(
          'pdfDocumentImage.jpg',
          Buffer.from(pdfDocumentImageBytes)
        );
      }
    );

    const result = await this.openAiApiClient.chat.completions.create({
      model: 'gpt-4-vision-preview',
      // response_format: { type: "json_object" }, // not supported for now
      messages: [
        { role: 'system', content: optionsArg.systemMessage },
        ...optionsArg.messageHistory,
        {
          role: 'user',
          content: [
            { type: 'text', text: optionsArg.userMessage },
            ...(() => {
              const returnArray = [];
              for (const imageBytes of pdfDocumentImageBytesArray) {
                returnArray.push({
                  type: 'image_url',
                  image_url: {
                    url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
                  },
                });
              }
              return returnArray;
            })(),
          ],
        },
      ],
    });
    return {
      message: result.choices[0].message,
    };
  }
}