0.0.17

npmextra.json

@@ -5,14 +5,28 @@
       "githost": "code.foss.global",
       "gitscope": "push.rocks",
       "gitrepo": "smartai",
-      "description": "a standardaized interface to talk to AI models",
+      "description": "A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.",
       "npmPackagename": "@push.rocks/smartai",
       "license": "MIT",
-      "projectDomain": "push.rocks"
+      "projectDomain": "push.rocks",
+      "keywords": [
+        "AI integration",
+        "chatbot",
+        "TypeScript",
+        "OpenAI",
+        "Anthropic",
+        "multi-model support",
+        "audio responses",
+        "text-to-speech",
+        "streaming chat"
+      ]
     }
   },
   "npmci": {
     "npmGlobalTools": [],
     "npmAccessLevel": "public"
+  },
+  "tsdoc": {
+    "legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH  \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
   }
 }

package.json

@@ -1,8 +1,8 @@
 {
   "name": "@push.rocks/smartai",
-  "version": "0.0.7",
+  "version": "0.0.17",
   "private": false,
-  "description": "a standardaized interface to talk to AI models",
+  "description": "A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
   "type": "module",
@@ -14,19 +14,24 @@
     "buildDocs": "(tsdoc)"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.25",
+    "@git.zone/tsbuild": "^2.1.76",
     "@git.zone/tsbundle": "^2.0.5",
     "@git.zone/tsrun": "^1.2.46",
-    "@git.zone/tstest": "^1.0.44",
-    "@push.rocks/tapbundle": "^5.0.15",
-    "@types/node": "^20.8.7"
+    "@git.zone/tstest": "^1.0.90",
+    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/tapbundle": "^5.0.23",
+    "@types/node": "^20.12.12"
   },
   "dependencies": {
-    "@push.rocks/qenv": "^6.0.5",
-    "@push.rocks/smartfile": "^11.0.4",
-    "@push.rocks/smartpath": "^5.0.11",
+    "@anthropic-ai/sdk": "^0.21.0",
+    "@push.rocks/smartarray": "^1.0.8",
+    "@push.rocks/smartfile": "^11.0.14",
+    "@push.rocks/smartpath": "^5.0.18",
+    "@push.rocks/smartpdf": "^3.1.6",
     "@push.rocks/smartpromise": "^4.0.3",
-    "openai": "^4.31.0"
+    "@push.rocks/smartrequest": "^2.0.22",
+    "@push.rocks/webstream": "^1.0.8",
+    "openai": "^4.47.1"
   },
   "repository": {
     "type": "git",
@@ -50,5 +55,16 @@
     "cli.js",
     "npmextra.json",
     "readme.md"
+  ],
+  "keywords": [
+    "AI integration",
+    "chatbot",
+    "TypeScript",
+    "OpenAI",
+    "Anthropic",
+    "multi-model support",
+    "audio responses",
+    "text-to-speech",
+    "streaming chat"
   ]
 }

qenv.yml

@@ -1,2 +1,4 @@
 required:
-  - OPENAI_TOKEN
+  - OPENAI_TOKEN
+  - ANTHROPIC_TOKEN
+  - PERPLEXITY_TOKEN

readme.hints.md

@@ -0,0 +1 @@
+ 

readme.md

@@ -1,31 +1,111 @@
 # @push.rocks/smartai
-a standardaized interface to talk to AI models
 
-## Availabililty and Links
-* [npmjs.org (npm package)](https://www.npmjs.com/package/@push.rocks/smartai)
-* [gitlab.com (source)](https://code.foss.global/push.rocks/smartai)
-* [github.com (source mirror)](https://github.com/push.rocks/smartai)
-* [docs (typedoc)](https://push.rocks.gitlab.io/smartai/)
+Provides a standardized interface for integrating and conversing with multiple AI models, supporting operations like chat and potentially audio responses.
 
-## Status for master
+## Install
 
-Status Category | Status Badge
--- | --
-GitLab Pipelines | [![pipeline status](https://code.foss.global/push.rocks/smartai/badges/master/pipeline.svg)](https://lossless.cloud)
-GitLab Pipline Test Coverage | [![coverage report](https://code.foss.global/push.rocks/smartai/badges/master/coverage.svg)](https://lossless.cloud)
-npm | [![npm downloads per month](https://badgen.net/npm/dy/@push.rocks/smartai)](https://lossless.cloud)
-Snyk | [![Known Vulnerabilities](https://badgen.net/snyk/push.rocks/smartai)](https://lossless.cloud)
-TypeScript Support | [![TypeScript](https://badgen.net/badge/TypeScript/>=%203.x/blue?icon=typescript)](https://lossless.cloud)
-node Support | [![node](https://img.shields.io/badge/node->=%2010.x.x-blue.svg)](https://nodejs.org/dist/latest-v10.x/docs/api/)
-Code Style | [![Code Style](https://badgen.net/badge/style/prettier/purple)](https://lossless.cloud)
-PackagePhobia (total standalone install weight) | [![PackagePhobia](https://badgen.net/packagephobia/install/@push.rocks/smartai)](https://lossless.cloud)
-PackagePhobia (package size on registry) | [![PackagePhobia](https://badgen.net/packagephobia/publish/@push.rocks/smartai)](https://lossless.cloud)
-BundlePhobia (total size when bundled) | [![BundlePhobia](https://badgen.net/bundlephobia/minzip/@push.rocks/smartai)](https://lossless.cloud)
+To add @push.rocks/smartai to your project, run the following command in your terminal:
+
+```bash
+npm install @push.rocks/smartai
+```
+
+This command installs the package and adds it to your project's dependencies.
 
 ## Usage
-Use TypeScript for best in class intellisense
-For further information read the linked docs at the top of this readme.
 
-## Legal
-> MIT licensed | **©** [Task Venture Capital GmbH](https://task.vc)
-| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
+The `@push.rocks/smartai` package is a comprehensive solution for integrating and interacting with various AI models, designed to support operations ranging from chat interactions to possibly handling audio responses. This documentation will guide you through the process of utilizing `@push.rocks/smartai` in your applications, focusing on TypeScript and ESM syntax to demonstrate its full capabilities.
+
+### Getting Started
+
+Before you begin, ensure you have installed the package in your project as described in the **Install** section above. Once installed, you can start integrating AI functionalities into your application.
+
+### Initializing SmartAi
+
+The first step is to import and initialize the `SmartAi` class with appropriate options, including tokens for the AI services you plan to use:
+
+```typescript
+import { SmartAi } from '@push.rocks/smartai';
+
+const smartAi = new SmartAi({
+  openaiToken: 'your-openai-access-token',
+  anthropicToken: 'your-anthropic-access-token'
+});
+
+await smartAi.start();
+```
+
+### Creating Conversations with AI
+
+`SmartAi` provides a flexible interface to create and manage conversations with different AI providers. You can create a conversation with any supported AI provider like OpenAI or Anthropic by specifying the provider you want to use:
+
+```typescript
+const openAiConversation = await smartAi.createConversation('openai');
+const anthropicConversation = await smartAi.createConversation('anthropic');
+```
+
+### Chatting with AI
+
+Once you have a conversation instance, you can start sending messages to the AI and receive responses. Each conversation object provides methods to interact in a synchronous or asynchronous manner, depending on your use case.
+
+#### Synchronous Chat Example
+
+Here's how you can have a synchronous chat with OpenAI:
+
+```typescript
+const response = await openAiConversation.chat({
+  systemMessage: 'This is a greeting from the system.',
+  userMessage: 'Hello, AI! How are you today?',
+  messageHistory: [] // Previous messages in the conversation
+});
+
+console.log(response.message); // Log the response from AI
+```
+
+#### Streaming Chat Example
+
+For real-time, streaming interactions, you can utilize the streaming capabilities provided by the conversation object. This enables a continuous exchange of messages between your application and the AI:
+
+```typescript
+const inputStreamWriter = openAiConversation.getInputStreamWriter();
+const outputStream = openAiConversation.getOutputStream();
+
+inputStreamWriter.write('Hello, AI! Can you stream responses?');
+
+const reader = outputStream.getReader();
+reader.read().then(function processText({done, value}) {
+  if (done) {
+    console.log('Stream finished.');
+    return;
+  }
+  console.log('AI says:', value);
+  reader.read().then(processText); // Continue reading messages
+});
+```
+
+### Extending Conversations
+
+The modular design of `@push.rocks/smartai` allows you to extend conversations with additional features, such as handling audio responses or integrating other AI-powered functionalities. Utilize the provided AI providers' APIs to explore and implement a wide range of AI interactions within your conversations.
+
+### Conclusion
+
+With `@push.rocks/smartai`, integrating AI functionalities into your applications becomes streamlined and efficient. By leveraging the standardized interface provided by the package, you can easily converse with multiple AI models, expanding the capabilities of your applications with cutting-edge AI features. Whether you're implementing simple chat interactions or complex, real-time communication flows, `@push.rocks/smartai` offers the tools and flexibility needed to create engaging, AI-enhanced experiences.
+
+## License and Legal Information
+
+This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. 
+
+**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
+
+### Trademarks
+
+This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.
+
+### Company Information
+
+Task Venture Capital GmbH  
+Registered at District court Bremen HRB 35230 HB, Germany
+
+For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
+
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

test/test.ts

@@ -1,8 +1,84 @@
 import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
-import * as smartai from '../ts/index.js'
+import * as qenv from '@push.rocks/qenv';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as smartfile from '@push.rocks/smartfile';
 
-tap.test('first test', async () => {
-  console.log(smartai)
+const testQenv = new qenv.Qenv('./', './.nogit/');
+
+import * as smartai from '../ts/index.js';
+
+let testSmartai: smartai.SmartAi;
+
+tap.test('should create a smartai instance', async () => {
+  testSmartai = new smartai.SmartAi({
+    openaiToken: await testQenv.getEnvVarOnDemand('OPENAI_TOKEN'),
+  });
+  await testSmartai.start();
+});
+
+tap.test('should create chat response with openai', async () => {
+  const userMessage = 'How are you?';
+  const response = await testSmartai.openaiProvider.chat({
+    systemMessage: 'Hello',
+    userMessage: userMessage,
+    messageHistory: [
+    ],
+  });
+  console.log(`userMessage: ${userMessage}`);
+  console.log(response.message);
+});
+
+tap.test('should document a pdf', async () => {
+  const pdfUrl = 'https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf';
+  const pdfResponse = await smartrequest.getBinary(pdfUrl);
+  const result = await testSmartai.openaiProvider.document({
+    systemMessage: 'Classify the document. Only the following answers are allowed: "invoice", "bank account statement", "contract", "other"',
+    userMessage: "Classify the document.",
+    messageHistory: [],
+    pdfDocuments: [pdfResponse.body],
+  });
+  console.log(result);
+});
+
+tap.test('should recognize companies in a pdf', async () => {
+  const pdfBuffer = await smartfile.fs.toBuffer('./.nogit/demo_without_textlayer.pdf');
+  const result = await testSmartai.openaiProvider.document({
+    systemMessage: `
+      summarize the document.
+
+      answer in JSON format, adhering to the following schema:
+      \`\`\`typescript
+        type TAnswer = {
+          entitySender: {
+            type: 'official state entity' | 'company' | 'person';
+            name: string;
+            address: string;
+            city: string;
+            country: string;
+            EU: boolean; // wether the entity is within EU
+          };
+          entityReceiver: {
+            type: 'official state entity' | 'company' | 'person';
+            name: string;
+            address: string;
+            city: string;
+            country: string;
+            EU: boolean; // wether the entity is within EU
+          };
+          date: string; // the date of the document as YYYY-MM-DD
+          title: string; // a short title, suitable for a filename
+        }
+      \`\`\`
+    `,
+    userMessage: "Classify the document.",
+    messageHistory: [],
+    pdfDocuments: [pdfBuffer],
+  });
+  console.log(result);
 })
 
-tap.start()
+tap.test('should stop the smartai instance', async () => {
+  await testSmartai.stop();
+});
+
+export default tap.start();

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartai',
-  version: '0.0.7',
-  description: 'a standardaized interface to talk to AI models'
+  version: '0.0.17',
+  description: 'A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.'
 }

ts/abstract.classes.multimodal.ts

@@ -1,8 +1,32 @@
-export abstract class MultiModal {
+export abstract class MultiModalModel {
+  /**
+   * starts the model
+   */
   abstract start(): Promise<void>;
-  abstract stop(): Promise<void>;
   
-  // Defines a streaming interface for chat interactions.
-  // The implementation will vary based on the specific AI model.
-  abstract chatStream(input: ReadableStream<string>): ReadableStream<string>;
+  /**
+   * stops the model
+   */
+  abstract stop(): Promise<void>;
+
+  public abstract chat(optionsArg: {
+    systemMessage: string,
+    userMessage: string,
+    messageHistory: {
+      role: 'assistant' | 'user';
+      content: string;
+    }[]
+  }): Promise<{
+    role: 'assistant';
+    message: string;
+  }>
+  
+  /**
+   * Defines a streaming interface for chat interactions.
+   * The implementation will vary based on the specific AI model.
+   * @param input
+   */
+  public abstract chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>>;
+
+
 }

ts/classes.conversation.ts

@@ -0,0 +1,116 @@
+import type { SmartAi } from "./classes.smartai.js";
+import { OpenAiProvider } from "./provider.openai.js";
+
+type TProcessFunction = (input: string) => Promise<string>;
+
+export interface IConversationOptions {
+  processFunction: TProcessFunction;
+}
+
+/**
+ * a conversation
+ */
+export class Conversation {
+  // STATIC
+  public static async createWithOpenAi(smartaiRefArg: SmartAi) {
+    if (!smartaiRefArg.openaiProvider) {
+      throw new Error('OpenAI provider not available');
+    }
+    const conversation = new Conversation(smartaiRefArg, {
+      processFunction: async (input) => {
+        return '' // TODO implement proper streaming
+      }
+    });
+    return conversation;
+  }
+
+  public static async createWithAnthropic(smartaiRefArg: SmartAi) {
+    if (!smartaiRefArg.anthropicProvider) {
+      throw new Error('Anthropic provider not available');
+    }
+    const conversation = new Conversation(smartaiRefArg, {
+      processFunction: async (input) => {
+        return '' // TODO implement proper streaming
+      }
+    });
+    return conversation;
+  }
+
+  public static async createWithPerplexity(smartaiRefArg: SmartAi) {
+    if (!smartaiRefArg.perplexityProvider) {
+      throw new Error('Perplexity provider not available');
+    }
+    const conversation = new Conversation(smartaiRefArg, {
+      processFunction: async (input) => {
+        return '' // TODO implement proper streaming
+      }
+    });
+    return conversation;
+  }
+
+  public static async createWithOllama(smartaiRefArg: SmartAi) {
+    if (!smartaiRefArg.ollamaProvider) {
+      throw new Error('Ollama provider not available');
+    }
+    const conversation = new Conversation(smartaiRefArg, {
+      processFunction: async (input) => {
+        return '' // TODO implement proper streaming
+      }
+    });
+    return conversation;
+  }
+
+  // INSTANCE
+  smartaiRef: SmartAi
+  private systemMessage: string;
+  private processFunction: TProcessFunction;
+  private inputStreamWriter: WritableStreamDefaultWriter<string> | null = null;
+  private outputStreamController: ReadableStreamDefaultController<string> | null = null;
+
+  constructor(smartairefArg: SmartAi, options: IConversationOptions) {
+    this.processFunction = options.processFunction;
+  }
+
+  public async setSystemMessage(systemMessageArg: string) {
+    this.systemMessage = systemMessageArg;
+  }
+
+  private setupOutputStream(): ReadableStream<string> {
+    return new ReadableStream<string>({
+      start: (controller) => {
+        this.outputStreamController = controller;
+      }
+    });
+  }
+
+  private setupInputStream(): WritableStream<string> {
+    const writableStream = new WritableStream<string>({
+      write: async (chunk) => {
+        const processedData = await this.processFunction(chunk);
+        if (this.outputStreamController) {
+          this.outputStreamController.enqueue(processedData);
+        }
+      },
+      close: () => {
+        this.outputStreamController?.close();
+      },
+      abort: (err) => {
+        console.error('Stream aborted', err);
+        this.outputStreamController?.error(err);
+      }
+    });
+    return writableStream;
+  }
+
+  public getInputStreamWriter(): WritableStreamDefaultWriter<string> {
+    if (!this.inputStreamWriter) {
+      const inputStream = this.setupInputStream();
+      this.inputStreamWriter = inputStream.getWriter();
+    }
+    return this.inputStreamWriter;
+  }
+
+  public getOutputStream(): ReadableStream<string> {
+    return this.setupOutputStream();
+  }
+}

ts/classes.smartai.ts

@@ -0,0 +1,62 @@
+import { Conversation } from './classes.conversation.js';
+import * as plugins from './plugins.js';
+import { AnthropicProvider } from './provider.anthropic.js';
+import type { OllamaProvider } from './provider.ollama.js';
+import { OpenAiProvider } from './provider.openai.js';
+import type { PerplexityProvider } from './provider.perplexity.js';
+
+
+export interface ISmartAiOptions {
+  openaiToken?: string;
+  anthropicToken?: string;
+  perplexityToken?: string;
+}
+
+export type TProvider = 'openai' | 'anthropic' | 'perplexity' | 'ollama';
+
+export class SmartAi {
+  public options: ISmartAiOptions;
+
+  public openaiProvider: OpenAiProvider;
+  public anthropicProvider: AnthropicProvider;
+  public perplexityProvider: PerplexityProvider;
+  public ollamaProvider: OllamaProvider;
+
+  constructor(optionsArg: ISmartAiOptions) {
+    this.options = optionsArg;
+  }
+
+  public async start() {
+    if (this.options.openaiToken) {
+      this.openaiProvider = new OpenAiProvider({
+        openaiToken: this.options.openaiToken,
+      });
+      await this.openaiProvider.start();
+    }
+    if (this.options.anthropicToken) {
+      this.anthropicProvider = new AnthropicProvider({
+        anthropicToken: this.options.anthropicToken,
+      });
+    }
+  }
+
+  public async stop() {}
+  
+  /**
+   * create a new conversation
+   */
+  createConversation(provider: TProvider) {
+    switch (provider) {
+      case 'openai':
+        return Conversation.createWithOpenAi(this);
+      case 'anthropic':
+        return Conversation.createWithAnthropic(this);
+      case 'perplexity':
+        return Conversation.createWithPerplexity(this);
+      case 'ollama':
+        return Conversation.createWithOllama(this);
+      default:
+        throw new Error('Provider not available');
+    }
+  }
+}

ts/classes.tts.ts

@@ -0,0 +1,15 @@
+import type { SmartAi } from './classes.smartai.js';
+import * as plugins from './plugins.js';
+
+export class TTS {
+  public static async createWithOpenAi(smartaiRef: SmartAi): Promise<TTS> {
+    return new TTS(smartaiRef);
+  }
+
+  // INSTANCE
+  smartaiRef: SmartAi;
+
+  constructor(smartairefArg: SmartAi) {
+    this.smartaiRef = smartairefArg;
+  }
+}

ts/index.ts

@@ -1,3 +1,3 @@
-export * from './smartai.classes.smartai.js';
+export * from './classes.smartai.js';
 export * from './abstract.classes.multimodal.js';
 export * from './provider.openai.js';

ts/plugins.ts

@@ -7,20 +7,30 @@ export {
 
 // @push.rocks scope
 import * as qenv from '@push.rocks/qenv';
-import * as smartpath from '@push.rocks/smartpath';
-import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartarray from '@push.rocks/smartarray';
 import * as smartfile from '@push.rocks/smartfile';
+import * as smartpath from '@push.rocks/smartpath';
+import * as smartpdf from '@push.rocks/smartpdf';
+import * as smartpromise from '@push.rocks/smartpromise';
+import * as smartrequest from '@push.rocks/smartrequest';
+import * as webstream from '@push.rocks/webstream';
 
 export {
+  smartarray,
   qenv,
-  smartpath,
-  smartpromise,
   smartfile,
+  smartpath,
+  smartpdf,
+  smartpromise,
+  smartrequest,
+  webstream,
 }
 
 // third party
+import * as anthropic from '@anthropic-ai/sdk';
 import * as openai from 'openai';
 
 export {
+  anthropic,
   openai,
 }

ts/provider.anthropic.ts

@@ -0,0 +1,62 @@
+import * as plugins from './plugins.js';
+import * as paths from './paths.js';
+import { MultiModalModel } from './abstract.classes.multimodal.js';
+
+export interface IAnthropicProviderOptions {
+  anthropicToken: string;
+}
+
+export class AnthropicProvider extends MultiModalModel {
+  private options: IAnthropicProviderOptions;
+  public anthropicApiClient: plugins.anthropic.default;
+
+  constructor(optionsArg: IAnthropicProviderOptions) {
+    super();
+    this.options = optionsArg // Ensure the token is stored
+  }
+
+  async start() {
+    this.anthropicApiClient = new plugins.anthropic.default({
+      apiKey: this.options.anthropicToken,
+    });
+  }
+
+  async stop() {}
+
+  public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
+    // TODO: implement for OpenAI
+
+    const returnStream = new ReadableStream();
+    return returnStream;
+  }
+
+  // Implementing the synchronous chat interaction
+  public async chat(optionsArg: {
+    systemMessage: string;
+    userMessage: string;
+    messageHistory: {
+      role: 'assistant' | 'user';
+      content: string;
+    }[];
+  }) {
+    const result = await this.anthropicApiClient.messages.create({
+      model: 'claude-3-opus-20240229',
+      system: optionsArg.systemMessage,
+      messages: [
+        ...optionsArg.messageHistory,
+        { role: 'user', content: optionsArg.userMessage },
+      ],
+      max_tokens: 4000,
+    });
+
+    return {
+      role: result.role as 'assistant',
+      message: result.content.join('\n'),
+    };
+  }
+
+  private async audio(messageArg: string) {
+    // Anthropic does not provide an audio API, so this method is not implemented.
+    throw new Error('Audio generation is not yet supported by Anthropic.');
+  }
+}

ts/provider.ollama.ts

@@ -0,0 +1,3 @@
+import * as plugins from './plugins.js';
+
+export class OllamaProvider {}

ts/provider.openai.ts

@@ -1,88 +1,134 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 
-import { MultiModal } from './abstract.classes.multimodal.js';
+import { MultiModalModel } from './abstract.classes.multimodal.js';
 
-export class OpenAiProvider extends MultiModal {
-  private openAiToken: string;
+export interface IOpenaiProviderOptions {
+  openaiToken: string;
+}
+
+export class OpenAiProvider extends MultiModalModel {
+  private options: IOpenaiProviderOptions;
   public openAiApiClient: plugins.openai.default;
+  public smartpdfInstance: plugins.smartpdf.SmartPdf;
 
-  constructor(openaiToken: string) {
+  constructor(optionsArg: IOpenaiProviderOptions) {
     super();
-    this.openAiToken = openaiToken; // Ensure the token is stored
+    this.options = optionsArg;
   }
 
-  async start() {
+  public async start() {
     this.openAiApiClient = new plugins.openai.default({
-      apiKey: this.openAiToken,
+      apiKey: this.options.openaiToken,
       dangerouslyAllowBrowser: true,
     });
+    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
   }
 
-  async stop() {}
+  public async stop() {}
 
-  chatStream(input: ReadableStream<string>): ReadableStream<string> {
-    const decoder = new TextDecoder();
-    let messageHistory: { role: 'assistant' | 'user'; content: string }[] = [];
+  public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
+    // TODO: implement for OpenAI
 
-    return new ReadableStream({
-      async start(controller) {
-        const reader = input.getReader();
-        try {
-          let done, value;
-          while ((({ done, value } = await reader.read()), !done)) {
-            const userMessage = decoder.decode(value, { stream: true });
-            messageHistory.push({ role: 'user', content: userMessage });
-
-            const aiResponse = await this.chat('', userMessage, messageHistory);
-            messageHistory.push({ role: 'assistant', content: aiResponse.message });
-
-            // Directly enqueue the string response instead of encoding it first
-            controller.enqueue(aiResponse.message);
-          }
-          controller.close();
-        } catch (err) {
-          controller.error(err);
-        }
-      },
-    });
+    const returnStream = new ReadableStream();
+    return returnStream;
   }
 
   // Implementing the synchronous chat interaction
-  public async chat(
-    systemMessage: string,
-    userMessage: string,
+  public async chat(optionsArg: {
+    systemMessage: string;
+    userMessage: string;
     messageHistory: {
       role: 'assistant' | 'user';
       content: string;
-    }[]
-  ) {
+    }[];
+  }) {
     const result = await this.openAiApiClient.chat.completions.create({
-      model: 'gpt-3.5-turbo-16k-0613',
+      model: 'gpt-4o',
+
       messages: [
-        { role: 'system', content: systemMessage },
-        ...messageHistory,
-        { role: 'user', content: userMessage },
+        { role: 'system', content: optionsArg.systemMessage },
+        ...optionsArg.messageHistory,
+        { role: 'user', content: optionsArg.userMessage },
+      ],
+    });
+    return {
+      role: result.choices[0].message.role as 'assistant',
+      message: result.choices[0].message.content,
+    };
+  }
+
+  public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
+    const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
+    const result = await this.openAiApiClient.audio.speech.create({
+      model: 'tts-1-hd',
+      input: optionsArg.message,
+      voice: 'nova',
+      response_format: 'mp3',
+      speed: 1,
+    });
+    const stream = result.body;
+    done.resolve(stream);
+    return done.promise;
+  }
+
+  public async document(optionsArg: {
+    systemMessage: string;
+    userMessage: string;
+    pdfDocuments: Uint8Array[];
+    messageHistory: {
+      role: 'assistant' | 'user';
+      content: any;
+    }[];
+  }) {
+    let pdfDocumentImageBytesArray: Uint8Array[] = [];
+
+    for (const pdfDocument of optionsArg.pdfDocuments) {
+      const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
+      pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
+    }
+
+    console.log(`image smartfile array`);
+    console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
+
+    const smartfileArray = await plugins.smartarray.map(
+      pdfDocumentImageBytesArray,
+      async (pdfDocumentImageBytes) => {
+        return plugins.smartfile.SmartFile.fromBuffer(
+          'pdfDocumentImage.jpg',
+          Buffer.from(pdfDocumentImageBytes)
+        );
+      }
+    );
+
+    const result = await this.openAiApiClient.chat.completions.create({
+      model: 'gpt-4o',
+      // response_format: { type: "json_object" }, // not supported for now
+      messages: [
+        { role: 'system', content: optionsArg.systemMessage },
+        ...optionsArg.messageHistory,
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: optionsArg.userMessage },
+            ...(() => {
+              const returnArray = [];
+              for (const imageBytes of pdfDocumentImageBytesArray) {
+                returnArray.push({
+                  type: 'image_url',
+                  image_url: {
+                    url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
+                  },
+                });
+              }
+              return returnArray;
+            })(),
+          ],
+        },
       ],
     });
     return {
       message: result.choices[0].message,
     };
   }
-
-  public async audio(messageArg: string) {
-    const done = plugins.smartpromise.defer();
-    const result = await this.openAiApiClient.audio.speech.create({
-      model: 'tts-1-hd',
-      input: messageArg,
-      voice: 'nova',
-      response_format: 'mp3',
-      speed: 1,
-    });
-    const stream = result.body.pipe(plugins.smartfile.fsStream.createWriteStream(plugins.path.join(paths.nogitDir, 'output.mp3')));
-    stream.on('finish', () => {
-      done.resolve();
-    });
-    return done.promise;
-  }
 }

ts/provider.perplexity.ts

@@ -0,0 +1,3 @@
+import * as plugins from './plugins.js';
+
+export class PerplexityProvider {}

ts/smartai.classes.smartai.ts

@@ -1,53 +0,0 @@
-type TProcessFunction = (input: string) => Promise<string>;
-
-export interface ISmartAiOptions {
-  processFunction: TProcessFunction;
-}
-
-export class SmartAi {
-  private processFunction: TProcessFunction;
-  private inputStreamWriter: WritableStreamDefaultWriter<string> | null = null;
-  private outputStreamController: ReadableStreamDefaultController<string> | null = null;
-
-  constructor(options: ISmartAiOptions) {
-    this.processFunction = options.processFunction;
-  }
-
-  private setupOutputStream(): ReadableStream<string> {
-    return new ReadableStream<string>({
-      start: (controller) => {
-        this.outputStreamController = controller;
-      }
-    });
-  }
-
-  private setupInputStream(): WritableStream<string> {
-    return new WritableStream<string>({
-      write: async (chunk) => {
-        const processedData = await this.processFunction(chunk);
-        if (this.outputStreamController) {
-          this.outputStreamController.enqueue(processedData);
-        }
-      },
-      close: () => {
-        this.outputStreamController?.close();
-      },
-      abort: (err) => {
-        console.error('Stream aborted', err);
-        this.outputStreamController?.error(err);
-      }
-    });
-  }
-
-  public getInputStreamWriter(): WritableStreamDefaultWriter<string> {
-    if (!this.inputStreamWriter) {
-      const inputStream = this.setupInputStream();
-      this.inputStreamWriter = inputStream.getWriter();
-    }
-    return this.inputStreamWriter;
-  }
-
-  public getOutputStream(): ReadableStream<string> {
-    return this.setupOutputStream();
-  }
-}