fix(core): update

package.json

@@ -24,12 +24,14 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.20.7",
-    "@push.rocks/smartexpose": "^1.0.5",
+    "@push.rocks/smartarray": "^1.0.8",
     "@push.rocks/smartfile": "^11.0.14",
     "@push.rocks/smartpath": "^5.0.18",
+    "@push.rocks/smartpdf": "^3.1.5",
     "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartrequest": "^2.0.22",
     "@push.rocks/webstream": "^1.0.8",
-    "openai": "^4.38.3"
+    "openai": "^4.38.5"
   },
   "repository": {
     "type": "git",

test/test.ts

@@ -1,5 +1,7 @@
 import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
 import * as qenv from '@push.rocks/qenv';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartfile from '@push.rocks/smartfile';
 
 const testQenv = new qenv.Qenv('./', './.nogit/');
 
@@ -10,8 +12,83 @@ let testSmartai: smartai.SmartAi;
 tap.test('should create a smartai instance', async () => {
   testSmartai = new smartai.SmartAi({
     openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
-    
+    exposeCredentials: {
+      exposedBaseUrl: await testQenv.getEnvVarOnDemand('EXPOSED_BASE_URL'),
+      webdav: {
+        webdavCredentials: {
+          password: await testQenv.getEnvVarOnDemand('WEBDAV_SERVER_TOKEN'),
+          serverUrl: await testQenv.getEnvVarOnDemand('WEBDAV_SERVER_URL'),
+        },
+        webdavSubPath: 'smartai'
+      }
+    }
   });
+  await testSmartai.start();
+});
+
+tap.test('should create chat response with openai', async () => {
+  const userMessage = 'How are you?';
+  const response = await testSmartai.openaiProvider.chat({
+    systemMessage: 'Hello',
+    userMessage: userMessage,
+    messageHistory: [
+    ],
+  });
+  console.log(`userMessage: ${userMessage}`);
+  console.log(response.message.content);
+});
+
+tap.test('should document a pdf', async () => {
+  const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
+  const pdfResponse = await smartrequest.getBinary(pdfUrl);
+  const result = await testSmartai.openaiProvider.document({
+    systemMessage: 'Classify the document. Only the following answers are allowed: "invoice", "bank account statement", "contract", "other"',
+    userMessage: "Classify the document.",
+    messageHistory: [],
+    pdfDocuments: [pdfResponse.body],
+  });
+  console.log(result);
+});
+
+tap.test('should recognize companies in a pdf', async () => {
+  const pdfBuffer = await smartfile.fs.toBuffer('./.nogit/demo_without_textlayer.pdf');
+  const result = await testSmartai.openaiProvider.document({
+    systemMessage: `
+      summarize the document.
+
+      answer in JSON format, adhering to the following schema:
+      \`\`\`typescript
+        type TAnswer = {
+          entitySender: {
+            type: 'official state entity' | 'company' | 'person';
+            name: string;
+            address: string;
+            city: string;
+            country: string;
+            EU: boolean; // wether the entity is within EU
+          };
+          entityReceiver: {
+            type: 'official state entity' | 'company' | 'person';
+            name: string;
+            address: string;
+            city: string;
+            country: string;
+            EU: boolean; // wether the entity is within EU
+          };
+          date: string; // the date of the document as YYYY-MM-DD
+          title: string; // a short title, suitable for a filename
+        }
+      \`\`\`
+    `,
+    userMessage: "Classify the document.",
+    messageHistory: [],
+    pdfDocuments: [pdfBuffer],
+  });
+  console.log(result);
 })
 
-tap.start()
+tap.test('should stop the smartai instance', async () => {
+  await testSmartai.stop();
+});
+
+export default tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartai',
-  version: '0.0.10',
+  version: '0.0.11',
   description: 'Provides a standardized interface for integrating and conversing with multiple AI models, supporting operations like chat and potentially audio responses.'
 }

ts/classes.smartai.ts

@@ -2,7 +2,7 @@ import { Conversation } from './classes.conversation.js';
 import * as plugins from './plugins.js';
 import type { AnthropicProvider } from './provider.anthropic.js';
 import type { OllamaProvider } from './provider.ollama.js';
-import type { OpenAiProvider } from './provider.openai.js';
+import { OpenAiProvider } from './provider.openai.js';
 import type { PerplexityProvider } from './provider.perplexity.js';
 
 
@@ -10,7 +10,6 @@ export interface ISmartAiOptions {
   openaiToken?: string;
   anthropicToken?: string;
   perplexityToken?: string;
-  exposeCredentials?: plugins.smartexpose.ISmartExposeOptions; 
 }
 
 export class SmartAi {
@@ -26,7 +25,12 @@ export class SmartAi {
   }
 
   public async start() {
-
+    if (this.options.openaiToken) {
+      this.openaiProvider = new OpenAiProvider({
+        openaiToken: this.options.openaiToken,
+      });
+      await this.openaiProvider.start();
+    }
   }
 
   public async stop() {}

ts/plugins.ts

@@ -7,18 +7,22 @@ export {
 
 // @push.rocks scope
 import * as qenv from '@push.rocks/qenv';
-import * as smartexpose from '@push.rocks/smartexpose';
+import * as smartarray from '@push.rocks/smartarray';
-import * as smartpath from '@push.rocks/smartpath';
-import * as smartpromise from '@push.rocks/smartpromise';
 import * as smartfile from '@push.rocks/smartfile';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpdf from '@push.rocks/smartpdf';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrequest from '@push.rocks/smartrequest';
 import * as webstream from '@push.rocks/webstream';
 
 export {
+  smartarray,
   qenv,
-  smartexpose,
-  smartpath,
-  smartpromise,
   smartfile,
+  smartpath,
+  smartpdf,
+  smartpromise,
+  smartrequest,
   webstream,
 }
 

ts/provider.openai.ts

@@ -3,24 +3,29 @@ import * as paths from './paths.js';
 
 import { MultiModalModel } from './abstract.classes.multimodal.js';
 
-export class OpenAiProvider extends MultiModalModel {
+export interface IOpenaiProviderOptions {
-  public smartexposeInstance: plugins.smartexpose.SmartExpose;
+  openaiToken: string;
-  private openAiToken: string;
-  public openAiApiClient: plugins.openai.default;
-
-  constructor(openaiToken: string, expose) {
-    super();
-    this.openAiToken = openaiToken; // Ensure the token is stored
 }
 
-  async start() {
+export class OpenAiProvider extends MultiModalModel {
+  private options: IOpenaiProviderOptions;
+  public openAiApiClient: plugins.openai.default;
+  public smartpdfInstance: plugins.smartpdf.SmartPdf;
+
+  constructor(optionsArg: IOpenaiProviderOptions) {
+    super();
+    this.options = optionsArg;
+  }
+
+  public async start() {
     this.openAiApiClient = new plugins.openai.default({
-      apiKey: this.openAiToken,
+      apiKey: this.options.openaiToken,
       dangerouslyAllowBrowser: true,
     });
+    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
   }
 
-  async stop() {}
+  public async stop() {}
 
   public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
     // TODO: implement for OpenAI
@@ -30,16 +35,14 @@ export class OpenAiProvider extends MultiModalModel {
   }
 
   // Implementing the synchronous chat interaction
-  public async chat(
+  public async chat(optionsArg: {
-    optionsArg: {
+    systemMessage: string;
-      systemMessage: string,
+    userMessage: string;
-      userMessage: string,
     messageHistory: {
       role: 'assistant' | 'user';
       content: string;
-      }[]
+    }[];
-    }
+  }) {
-  ) {
     const result = await this.openAiApiClient.chat.completions.create({
       model: 'gpt-4-turbo-preview',
 
@@ -69,30 +72,58 @@ export class OpenAiProvider extends MultiModalModel {
   }
 
   public async document(optionsArg: {
-    systemMessage: string,
+    systemMessage: string;
-    userMessage: string,
+    userMessage: string;
-    documents: Uint8Array[],
+    pdfDocuments: Uint8Array[];
     messageHistory: {
       role: 'assistant' | 'user';
       content: any;
     }[];
   }) {
+    let pdfDocumentImageBytesArray: Uint8Array[] = [];
+
+    for (const pdfDocument of optionsArg.pdfDocuments) {
+      const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
+      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
+    }
+
+    console.log(`image smartfile array`);
+    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
+
+    const smartfileArray = await plugins.smartarray.map(
+      pdfDocumentImageBytesArray,
+      async (pdfDocumentImageBytes) => {
+        return plugins.smartfile.SmartFile.fromBuffer(
+          'pdfDocumentImage.jpg',
+          Buffer.from(pdfDocumentImageBytes)
+        );
+      }
+    );
+
     const result = await this.openAiApiClient.chat.completions.create({
       model: 'gpt-4-vision-preview',
-
+      // response_format: { type: "json_object" }, // not supported for now
       messages: [
         { role: 'system', content: optionsArg.systemMessage },
         ...optionsArg.messageHistory,
-        { role: 'user', content: [
+        {
+          role: 'user',
+          content: [
             { type: 'text', text: optionsArg.userMessage },
             ...(() => {
               const returnArray = [];
-            for (const document of optionsArg.documents) {
+              for (const imageBytes of pdfDocumentImageBytesArray) {
-              returnArray.push({type: 'image_url', image_url: })
+                returnArray.push({
+                  type: 'image_url',
+                  image_url: {
+                    url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
+                  },
+                });
               }
               return returnArray;
-          })()
+            })(),
-        ] },
+          ],
+        },
       ],
     });
     return {