smartai/ts/provider.openai.ts

135 lines
3.9 KiB
TypeScript
Raw Normal View History

2024-03-31 00:32:37 +00:00
import * as plugins from './plugins.js';
import * as paths from './paths.js';
2024-04-04 00:47:44 +00:00
import { MultiModalModel } from './abstract.classes.multimodal.js';
2024-03-31 00:32:37 +00:00
2024-04-27 10:47:49 +00:00
export interface IOpenaiProviderOptions {
openaiToken: string;
}
2024-04-04 00:47:44 +00:00
export class OpenAiProvider extends MultiModalModel {
2024-04-27 10:47:49 +00:00
private options: IOpenaiProviderOptions;
2024-03-31 00:32:37 +00:00
public openAiApiClient: plugins.openai.default;
2024-04-27 10:47:49 +00:00
public smartpdfInstance: plugins.smartpdf.SmartPdf;
2024-03-31 00:32:37 +00:00
2024-04-27 10:47:49 +00:00
constructor(optionsArg: IOpenaiProviderOptions) {
2024-03-31 00:32:37 +00:00
super();
2024-04-27 10:47:49 +00:00
this.options = optionsArg;
2024-03-31 00:32:37 +00:00
}
2024-04-27 10:47:49 +00:00
public async start() {
2024-03-31 00:32:37 +00:00
this.openAiApiClient = new plugins.openai.default({
2024-04-27 10:47:49 +00:00
apiKey: this.options.openaiToken,
2024-03-31 00:32:37 +00:00
dangerouslyAllowBrowser: true,
});
2024-04-27 10:47:49 +00:00
this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
2024-03-31 00:32:37 +00:00
}
2024-04-27 10:47:49 +00:00
public async stop() {}
2024-03-31 00:32:37 +00:00
2024-04-25 08:49:07 +00:00
public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
// TODO: implement for OpenAI
2024-03-31 00:32:37 +00:00
2024-04-25 08:49:07 +00:00
const returnStream = new ReadableStream();
return returnStream;
2024-03-31 00:32:37 +00:00
}
// Implementing the synchronous chat interaction
2024-04-27 10:47:49 +00:00
public async chat(optionsArg: {
systemMessage: string;
userMessage: string;
messageHistory: {
role: 'assistant' | 'user';
content: string;
}[];
}) {
2024-03-31 00:32:37 +00:00
const result = await this.openAiApiClient.chat.completions.create({
2024-05-17 14:25:22 +00:00
model: 'gpt-4o',
2024-04-25 08:49:07 +00:00
2024-03-31 00:32:37 +00:00
messages: [
2024-04-25 08:49:07 +00:00
{ role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory,
{ role: 'user', content: optionsArg.userMessage },
2024-03-31 00:32:37 +00:00
],
});
return {
role: result.choices[0].message.role as 'assistant',
message: result.choices[0].message.content,
2024-03-31 00:32:37 +00:00
};
}
2024-04-25 08:49:07 +00:00
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
2024-03-31 00:32:37 +00:00
const result = await this.openAiApiClient.audio.speech.create({
model: 'tts-1-hd',
2024-04-25 08:49:07 +00:00
input: optionsArg.message,
2024-03-31 00:32:37 +00:00
voice: 'nova',
response_format: 'mp3',
speed: 1,
});
2024-04-25 08:49:07 +00:00
const stream = result.body;
done.resolve(stream);
2024-03-31 00:32:37 +00:00
return done.promise;
}
2024-04-25 08:49:07 +00:00
public async document(optionsArg: {
2024-04-27 10:47:49 +00:00
systemMessage: string;
userMessage: string;
pdfDocuments: Uint8Array[];
2024-04-25 08:49:07 +00:00
messageHistory: {
role: 'assistant' | 'user';
content: any;
}[];
}) {
2024-04-27 10:47:49 +00:00
let pdfDocumentImageBytesArray: Uint8Array[] = [];
for (const pdfDocument of optionsArg.pdfDocuments) {
const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
}
console.log(`image smartfile array`);
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
const smartfileArray = await plugins.smartarray.map(
pdfDocumentImageBytesArray,
async (pdfDocumentImageBytes) => {
return plugins.smartfile.SmartFile.fromBuffer(
'pdfDocumentImage.jpg',
Buffer.from(pdfDocumentImageBytes)
);
}
);
2024-04-25 08:49:07 +00:00
const result = await this.openAiApiClient.chat.completions.create({
2024-05-17 14:25:22 +00:00
model: 'gpt-4o',
2024-04-27 10:47:49 +00:00
// response_format: { type: "json_object" }, // not supported for now
2024-04-25 08:49:07 +00:00
messages: [
{ role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory,
2024-04-27 10:47:49 +00:00
{
role: 'user',
content: [
{ type: 'text', text: optionsArg.userMessage },
...(() => {
const returnArray = [];
for (const imageBytes of pdfDocumentImageBytesArray) {
returnArray.push({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
},
});
}
return returnArray;
})(),
],
},
2024-04-25 08:49:07 +00:00
],
});
return {
message: result.choices[0].message,
};
}
2024-03-31 00:32:37 +00:00
}