fix(core): update

ts/provider.openai.ts

@@ -3,24 +3,29 @@ import * as paths from './paths.js';
 
 import { MultiModalModel } from './abstract.classes.multimodal.js';
 
-export class OpenAiProvider extends MultiModalModel {
-  public smartexposeInstance: plugins.smartexpose.SmartExpose;
-  private openAiToken: string;
-  public openAiApiClient: plugins.openai.default;
+export interface IOpenaiProviderOptions {
+  openaiToken: string;
+}
 
-  constructor(openaiToken: string, expose) {
+export class OpenAiProvider extends MultiModalModel {
+  private options: IOpenaiProviderOptions;
+  public openAiApiClient: plugins.openai.default;
+  public smartpdfInstance: plugins.smartpdf.SmartPdf;
+
+  constructor(optionsArg: IOpenaiProviderOptions) {
     super();
-    this.openAiToken = openaiToken; // Ensure the token is stored
+    this.options = optionsArg;
   }
 
-  async start() {
+  public async start() {
     this.openAiApiClient = new plugins.openai.default({
-      apiKey: this.openAiToken,
+      apiKey: this.options.openaiToken,
       dangerouslyAllowBrowser: true,
     });
+    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
   }
 
-  async stop() {}
+  public async stop() {}
 
   public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
     // TODO: implement for OpenAI
@@ -30,16 +35,14 @@ export class OpenAiProvider extends MultiModalModel {
   }
 
   // Implementing the synchronous chat interaction
-  public async chat(
-    optionsArg: {
-      systemMessage: string,
-      userMessage: string,
-      messageHistory: {
-        role: 'assistant' | 'user';
-        content: string;
-      }[]
-    }
-  ) {
+  public async chat(optionsArg: {
+    systemMessage: string;
+    userMessage: string;
+    messageHistory: {
+      role: 'assistant' | 'user';
+      content: string;
+    }[];
+  }) {
     const result = await this.openAiApiClient.chat.completions.create({
       model: 'gpt-4-turbo-preview',
 
@@ -69,30 +72,58 @@ export class OpenAiProvider extends MultiModalModel {
   }
 
   public async document(optionsArg: {
-    systemMessage: string,
-    userMessage: string,
-    documents: Uint8Array[],
+    systemMessage: string;
+    userMessage: string;
+    pdfDocuments: Uint8Array[];
     messageHistory: {
       role: 'assistant' | 'user';
       content: any;
     }[];
   }) {
+    let pdfDocumentImageBytesArray: Uint8Array[] = [];
+
+    for (const pdfDocument of optionsArg.pdfDocuments) {
+      const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
+      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
+    }
+
+    console.log(`image smartfile array`);
+    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
+
+    const smartfileArray = await plugins.smartarray.map(
+      pdfDocumentImageBytesArray,
+      async (pdfDocumentImageBytes) => {
+        return plugins.smartfile.SmartFile.fromBuffer(
+          'pdfDocumentImage.jpg',
+          Buffer.from(pdfDocumentImageBytes)
+        );
+      }
+    );
+
     const result = await this.openAiApiClient.chat.completions.create({
       model: 'gpt-4-vision-preview',
-
+      // response_format: { type: "json_object" }, // not supported for now
       messages: [
         { role: 'system', content: optionsArg.systemMessage },
         ...optionsArg.messageHistory,
-        { role: 'user', content: [
-          {type: 'text', text: optionsArg.userMessage},
-          ...(() => {
-            const returnArray = [];
-            for (const document of optionsArg.documents) {
-              returnArray.push({type: 'image_url', image_url: })
-            }
-            return returnArray;
-          })()
-        ] },
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: optionsArg.userMessage },
+            ...(() => {
+              const returnArray = [];
+              for (const imageBytes of pdfDocumentImageBytesArray) {
+                returnArray.push({
+                  type: 'image_url',
+                  image_url: {
+                    url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
+                  },
+                });
+              }
+              return returnArray;
+            })(),
+          ],
+        },
       ],
     });
     return {