135 lines
3.9 KiB
TypeScript
135 lines
3.9 KiB
TypeScript
import * as plugins from './plugins.js';
|
|
import * as paths from './paths.js';
|
|
|
|
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
|
|
|
export interface IOpenaiProviderOptions {
|
|
openaiToken: string;
|
|
}
|
|
|
|
export class OpenAiProvider extends MultiModalModel {
|
|
private options: IOpenaiProviderOptions;
|
|
public openAiApiClient: plugins.openai.default;
|
|
public smartpdfInstance: plugins.smartpdf.SmartPdf;
|
|
|
|
constructor(optionsArg: IOpenaiProviderOptions) {
|
|
super();
|
|
this.options = optionsArg;
|
|
}
|
|
|
|
public async start() {
|
|
this.openAiApiClient = new plugins.openai.default({
|
|
apiKey: this.options.openaiToken,
|
|
dangerouslyAllowBrowser: true,
|
|
});
|
|
this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
|
|
}
|
|
|
|
public async stop() {}
|
|
|
|
public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
|
|
// TODO: implement for OpenAI
|
|
|
|
const returnStream = new ReadableStream();
|
|
return returnStream;
|
|
}
|
|
|
|
// Implementing the synchronous chat interaction
|
|
public async chat(optionsArg: {
|
|
systemMessage: string;
|
|
userMessage: string;
|
|
messageHistory: {
|
|
role: 'assistant' | 'user';
|
|
content: string;
|
|
}[];
|
|
}) {
|
|
const result = await this.openAiApiClient.chat.completions.create({
|
|
model: 'gpt-4-turbo-preview',
|
|
|
|
messages: [
|
|
{ role: 'system', content: optionsArg.systemMessage },
|
|
...optionsArg.messageHistory,
|
|
{ role: 'user', content: optionsArg.userMessage },
|
|
],
|
|
});
|
|
return {
|
|
role: result.choices[0].message.role as 'assistant',
|
|
message: result.choices[0].message.content,
|
|
};
|
|
}
|
|
|
|
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
|
const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
|
|
const result = await this.openAiApiClient.audio.speech.create({
|
|
model: 'tts-1-hd',
|
|
input: optionsArg.message,
|
|
voice: 'nova',
|
|
response_format: 'mp3',
|
|
speed: 1,
|
|
});
|
|
const stream = result.body;
|
|
done.resolve(stream);
|
|
return done.promise;
|
|
}
|
|
|
|
public async document(optionsArg: {
|
|
systemMessage: string;
|
|
userMessage: string;
|
|
pdfDocuments: Uint8Array[];
|
|
messageHistory: {
|
|
role: 'assistant' | 'user';
|
|
content: any;
|
|
}[];
|
|
}) {
|
|
let pdfDocumentImageBytesArray: Uint8Array[] = [];
|
|
|
|
for (const pdfDocument of optionsArg.pdfDocuments) {
|
|
const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
|
|
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
|
|
}
|
|
|
|
console.log(`image smartfile array`);
|
|
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
|
|
|
|
const smartfileArray = await plugins.smartarray.map(
|
|
pdfDocumentImageBytesArray,
|
|
async (pdfDocumentImageBytes) => {
|
|
return plugins.smartfile.SmartFile.fromBuffer(
|
|
'pdfDocumentImage.jpg',
|
|
Buffer.from(pdfDocumentImageBytes)
|
|
);
|
|
}
|
|
);
|
|
|
|
const result = await this.openAiApiClient.chat.completions.create({
|
|
model: 'gpt-4-vision-preview',
|
|
// response_format: { type: "json_object" }, // not supported for now
|
|
messages: [
|
|
{ role: 'system', content: optionsArg.systemMessage },
|
|
...optionsArg.messageHistory,
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: optionsArg.userMessage },
|
|
...(() => {
|
|
const returnArray = [];
|
|
for (const imageBytes of pdfDocumentImageBytesArray) {
|
|
returnArray.push({
|
|
type: 'image_url',
|
|
image_url: {
|
|
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
|
|
},
|
|
});
|
|
}
|
|
return returnArray;
|
|
})(),
|
|
],
|
|
},
|
|
],
|
|
});
|
|
return {
|
|
message: result.choices[0].message,
|
|
};
|
|
}
|
|
}
|