fix(core): update

This commit is contained in:
2024-04-27 12:47:49 +02:00
parent 1ce412fd00
commit 158d49fa95
7 changed files with 294 additions and 318 deletions

View File

@@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartai',
version: '0.0.10',
version: '0.0.11',
description: 'Provides a standardized interface for integrating and conversing with multiple AI models, supporting operations like chat and potentially audio responses.'
}

View File

@@ -2,7 +2,7 @@ import { Conversation } from './classes.conversation.js';
import * as plugins from './plugins.js';
import type { AnthropicProvider } from './provider.anthropic.js';
import type { OllamaProvider } from './provider.ollama.js';
import type { OpenAiProvider } from './provider.openai.js';
import { OpenAiProvider } from './provider.openai.js';
import type { PerplexityProvider } from './provider.perplexity.js';
@@ -10,7 +10,6 @@ export interface ISmartAiOptions {
openaiToken?: string;
anthropicToken?: string;
perplexityToken?: string;
exposeCredentials?: plugins.smartexpose.ISmartExposeOptions;
}
export class SmartAi {
@@ -26,7 +25,12 @@ export class SmartAi {
}
public async start() {
if (this.options.openaiToken) {
this.openaiProvider = new OpenAiProvider({
openaiToken: this.options.openaiToken,
});
await this.openaiProvider.start();
}
}
public async stop() {}

View File

@@ -7,18 +7,22 @@ export {
// @push.rocks scope
import * as qenv from '@push.rocks/qenv';
import * as smartexpose from '@push.rocks/smartexpose';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartarray from '@push.rocks/smartarray';
import * as smartfile from '@push.rocks/smartfile';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpdf from '@push.rocks/smartpdf';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrequest from '@push.rocks/smartrequest';
import * as webstream from '@push.rocks/webstream';
export {
smartarray,
qenv,
smartexpose,
smartpath,
smartpromise,
smartfile,
smartpath,
smartpdf,
smartpromise,
smartrequest,
webstream,
}

View File

@@ -3,24 +3,29 @@ import * as paths from './paths.js';
import { MultiModalModel } from './abstract.classes.multimodal.js';
export class OpenAiProvider extends MultiModalModel {
public smartexposeInstance: plugins.smartexpose.SmartExpose;
private openAiToken: string;
public openAiApiClient: plugins.openai.default;
export interface IOpenaiProviderOptions {
openaiToken: string;
}
constructor(openaiToken: string, expose) {
export class OpenAiProvider extends MultiModalModel {
private options: IOpenaiProviderOptions;
public openAiApiClient: plugins.openai.default;
public smartpdfInstance: plugins.smartpdf.SmartPdf;
constructor(optionsArg: IOpenaiProviderOptions) {
super();
this.openAiToken = openaiToken; // Ensure the token is stored
this.options = optionsArg;
}
async start() {
public async start() {
this.openAiApiClient = new plugins.openai.default({
apiKey: this.openAiToken,
apiKey: this.options.openaiToken,
dangerouslyAllowBrowser: true,
});
this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
}
async stop() {}
public async stop() {}
public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
// TODO: implement for OpenAI
@@ -30,16 +35,14 @@ export class OpenAiProvider extends MultiModalModel {
}
// Implementing the synchronous chat interaction
public async chat(
optionsArg: {
systemMessage: string,
userMessage: string,
messageHistory: {
role: 'assistant' | 'user';
content: string;
}[]
}
) {
public async chat(optionsArg: {
systemMessage: string;
userMessage: string;
messageHistory: {
role: 'assistant' | 'user';
content: string;
}[];
}) {
const result = await this.openAiApiClient.chat.completions.create({
model: 'gpt-4-turbo-preview',
@@ -69,30 +72,58 @@ export class OpenAiProvider extends MultiModalModel {
}
public async document(optionsArg: {
systemMessage: string,
userMessage: string,
documents: Uint8Array[],
systemMessage: string;
userMessage: string;
pdfDocuments: Uint8Array[];
messageHistory: {
role: 'assistant' | 'user';
content: any;
}[];
}) {
let pdfDocumentImageBytesArray: Uint8Array[] = [];
for (const pdfDocument of optionsArg.pdfDocuments) {
const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
}
console.log(`image smartfile array`);
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
const smartfileArray = await plugins.smartarray.map(
pdfDocumentImageBytesArray,
async (pdfDocumentImageBytes) => {
return plugins.smartfile.SmartFile.fromBuffer(
'pdfDocumentImage.jpg',
Buffer.from(pdfDocumentImageBytes)
);
}
);
const result = await this.openAiApiClient.chat.completions.create({
model: 'gpt-4-vision-preview',
// response_format: { type: "json_object" }, // not supported for now
messages: [
{ role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory,
{ role: 'user', content: [
{type: 'text', text: optionsArg.userMessage},
...(() => {
const returnArray = [];
for (const document of optionsArg.documents) {
returnArray.push({type: 'image_url', image_url: })
}
return returnArray;
})()
] },
{
role: 'user',
content: [
{ type: 'text', text: optionsArg.userMessage },
...(() => {
const returnArray = [];
for (const imageBytes of pdfDocumentImageBytesArray) {
returnArray.push({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
},
});
}
return returnArray;
})(),
],
},
],
});
return {