458 lines
15 KiB
TypeScript
458 lines
15 KiB
TypeScript
import * as plugins from './plugins.js';
|
|
import * as paths from './paths.js';
|
|
import { Readable } from 'stream';
|
|
|
|
// Custom type definition for chat completion messages
|
|
export type TChatCompletionRequestMessage = {
|
|
role: "system" | "user" | "assistant";
|
|
content: string;
|
|
};
|
|
|
|
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
|
import type {
|
|
ResearchOptions,
|
|
ResearchResponse,
|
|
ImageGenerateOptions,
|
|
ImageEditOptions,
|
|
ImageResponse
|
|
} from './abstract.classes.multimodal.js';
|
|
|
|
export interface IOpenaiProviderOptions {
|
|
openaiToken: string;
|
|
chatModel?: string;
|
|
audioModel?: string;
|
|
visionModel?: string;
|
|
researchModel?: string;
|
|
imageModel?: string;
|
|
enableWebSearch?: boolean;
|
|
}
|
|
|
|
export class OpenAiProvider extends MultiModalModel {
|
|
private options: IOpenaiProviderOptions;
|
|
public openAiApiClient: plugins.openai.default;
|
|
|
|
constructor(optionsArg: IOpenaiProviderOptions) {
|
|
super();
|
|
this.options = optionsArg;
|
|
}
|
|
|
|
public async start() {
|
|
await super.start();
|
|
this.openAiApiClient = new plugins.openai.default({
|
|
apiKey: this.options.openaiToken,
|
|
dangerouslyAllowBrowser: true,
|
|
});
|
|
}
|
|
|
|
public async stop() {
|
|
await super.stop();
|
|
}
|
|
|
|
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
|
// Create a TextDecoder to handle incoming chunks
|
|
const decoder = new TextDecoder();
|
|
let buffer = '';
|
|
let currentMessage: {
|
|
role: "function" | "user" | "system" | "assistant" | "tool" | "developer";
|
|
content: string;
|
|
} | null = null;
|
|
|
|
// Create a TransformStream to process the input
|
|
const transform = new TransformStream<Uint8Array, string>({
|
|
transform: async (chunk, controller) => {
|
|
buffer += decoder.decode(chunk, { stream: true });
|
|
|
|
// Try to parse complete JSON messages from the buffer
|
|
while (true) {
|
|
const newlineIndex = buffer.indexOf('\n');
|
|
if (newlineIndex === -1) break;
|
|
|
|
const line = buffer.slice(0, newlineIndex);
|
|
buffer = buffer.slice(newlineIndex + 1);
|
|
|
|
if (line.trim()) {
|
|
try {
|
|
const message = JSON.parse(line);
|
|
currentMessage = {
|
|
role: (message.role || 'user') as "function" | "user" | "system" | "assistant" | "tool" | "developer",
|
|
content: message.content || '',
|
|
};
|
|
} catch (e) {
|
|
console.error('Failed to parse message:', e);
|
|
}
|
|
}
|
|
}
|
|
|
|
// If we have a complete message, send it to OpenAI
|
|
if (currentMessage) {
|
|
const messageToSend = { role: "user" as const, content: currentMessage.content };
|
|
const chatModel = this.options.chatModel ?? 'gpt-5-mini';
|
|
const requestParams: any = {
|
|
model: chatModel,
|
|
messages: [messageToSend],
|
|
stream: true,
|
|
};
|
|
// Temperature is omitted since the model does not support it.
|
|
const stream = await this.openAiApiClient.chat.completions.create(requestParams);
|
|
// Explicitly cast the stream as an async iterable to satisfy TypeScript.
|
|
const streamAsyncIterable = stream as unknown as AsyncIterableIterator<any>;
|
|
// Process each chunk from OpenAI
|
|
for await (const chunk of streamAsyncIterable) {
|
|
const content = chunk.choices[0]?.delta?.content;
|
|
if (content) {
|
|
controller.enqueue(content);
|
|
}
|
|
}
|
|
currentMessage = null;
|
|
}
|
|
},
|
|
|
|
flush(controller) {
|
|
if (buffer) {
|
|
try {
|
|
const message = JSON.parse(buffer);
|
|
controller.enqueue(message.content || '');
|
|
} catch (e) {
|
|
console.error('Failed to parse remaining buffer:', e);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
// Connect the input to our transform stream
|
|
return input.pipeThrough(transform);
|
|
}
|
|
|
|
// Implementing the synchronous chat interaction
|
|
public async chat(optionsArg: {
|
|
systemMessage: string;
|
|
userMessage: string;
|
|
messageHistory: {
|
|
role: 'assistant' | 'user';
|
|
content: string;
|
|
}[];
|
|
}) {
|
|
const chatModel = this.options.chatModel ?? 'gpt-5-mini';
|
|
const requestParams: any = {
|
|
model: chatModel,
|
|
messages: [
|
|
{ role: 'system', content: optionsArg.systemMessage },
|
|
...optionsArg.messageHistory,
|
|
{ role: 'user', content: optionsArg.userMessage },
|
|
],
|
|
};
|
|
// Temperature parameter removed to avoid unsupported error.
|
|
const result = await this.openAiApiClient.chat.completions.create(requestParams);
|
|
return {
|
|
role: result.choices[0].message.role as 'assistant',
|
|
message: result.choices[0].message.content,
|
|
};
|
|
}
|
|
|
|
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
|
const done = plugins.smartpromise.defer<NodeJS.ReadableStream>();
|
|
const result = await this.openAiApiClient.audio.speech.create({
|
|
model: this.options.audioModel ?? 'tts-1-hd',
|
|
input: optionsArg.message,
|
|
voice: 'nova',
|
|
response_format: 'mp3',
|
|
speed: 1,
|
|
});
|
|
const stream = result.body;
|
|
const nodeStream = Readable.fromWeb(stream as any);
|
|
done.resolve(nodeStream);
|
|
return done.promise;
|
|
}
|
|
|
|
public async document(optionsArg: {
|
|
systemMessage: string;
|
|
userMessage: string;
|
|
pdfDocuments: Uint8Array[];
|
|
messageHistory: {
|
|
role: 'assistant' | 'user';
|
|
content: any;
|
|
}[];
|
|
}) {
|
|
// Ensure SmartPdf is initialized before processing documents
|
|
await this.ensureSmartpdfReady();
|
|
|
|
let pdfDocumentImageBytesArray: Uint8Array[] = [];
|
|
|
|
// Convert each PDF into one or more image byte arrays.
|
|
for (const pdfDocument of optionsArg.pdfDocuments) {
|
|
const documentImageArray = await this.smartpdfInstance!.convertPDFToPngBytes(pdfDocument);
|
|
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
|
|
}
|
|
|
|
console.log(`image smartfile array`);
|
|
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
|
|
|
|
// Filter out any empty buffers to avoid sending invalid image URLs.
|
|
const validImageBytesArray = pdfDocumentImageBytesArray.filter(imageBytes => imageBytes && imageBytes.length > 0);
|
|
const imageAttachments = validImageBytesArray.map(imageBytes => ({
|
|
type: 'image_url',
|
|
image_url: {
|
|
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
|
|
},
|
|
}));
|
|
|
|
const chatModel = this.options.chatModel ?? 'gpt-5-mini';
|
|
const requestParams: any = {
|
|
model: chatModel,
|
|
messages: [
|
|
{ role: 'system', content: optionsArg.systemMessage },
|
|
...optionsArg.messageHistory,
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: optionsArg.userMessage },
|
|
...imageAttachments,
|
|
],
|
|
},
|
|
],
|
|
};
|
|
// Temperature parameter removed.
|
|
const result = await this.openAiApiClient.chat.completions.create(requestParams);
|
|
return {
|
|
message: result.choices[0].message,
|
|
};
|
|
}
|
|
|
|
public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
|
|
const visionModel = this.options.visionModel ?? '04-mini';
|
|
const requestParams: any = {
|
|
model: visionModel,
|
|
messages: [
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: optionsArg.prompt },
|
|
{
|
|
type: 'image_url',
|
|
image_url: {
|
|
url: `data:image/jpeg;base64,${optionsArg.image.toString('base64')}`
|
|
}
|
|
}
|
|
]
|
|
}
|
|
],
|
|
max_tokens: 300
|
|
};
|
|
const result = await this.openAiApiClient.chat.completions.create(requestParams);
|
|
return result.choices[0].message.content || '';
|
|
}
|
|
|
|
public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
|
|
// Determine which model to use - Deep Research API requires specific models
|
|
let model: string;
|
|
if (optionsArg.searchDepth === 'deep') {
|
|
model = this.options.researchModel || 'o4-mini-deep-research-2025-06-26';
|
|
} else {
|
|
// For basic/advanced, still use deep research models if web search is needed
|
|
if (optionsArg.includeWebSearch) {
|
|
model = this.options.researchModel || 'o4-mini-deep-research-2025-06-26';
|
|
} else {
|
|
model = this.options.chatModel || 'gpt-5-mini';
|
|
}
|
|
}
|
|
|
|
const systemMessage = 'You are a research assistant. Provide comprehensive answers with citations and sources when available.';
|
|
|
|
// Prepare request parameters using Deep Research API format
|
|
const requestParams: any = {
|
|
model,
|
|
instructions: systemMessage,
|
|
input: optionsArg.query
|
|
};
|
|
|
|
// Add web search tool if requested
|
|
if (optionsArg.includeWebSearch || optionsArg.searchDepth === 'deep') {
|
|
requestParams.tools = [
|
|
{
|
|
type: 'web_search_preview',
|
|
search_context_size: optionsArg.searchDepth === 'deep' ? 'high' :
|
|
optionsArg.searchDepth === 'advanced' ? 'medium' : 'low'
|
|
}
|
|
];
|
|
}
|
|
|
|
// Add background flag for deep research
|
|
if (optionsArg.background && optionsArg.searchDepth === 'deep') {
|
|
requestParams.background = true;
|
|
}
|
|
|
|
try {
|
|
// Execute the research request using Deep Research API
|
|
const result = await this.openAiApiClient.responses.create(requestParams);
|
|
|
|
// Extract the answer from output items
|
|
let answer = '';
|
|
const sources: Array<{ url: string; title: string; snippet: string }> = [];
|
|
const searchQueries: string[] = [];
|
|
|
|
// Process output items
|
|
for (const item of result.output || []) {
|
|
// Extract message content
|
|
if (item.type === 'message' && 'content' in item) {
|
|
const messageItem = item as any;
|
|
for (const contentItem of messageItem.content || []) {
|
|
if (contentItem.type === 'output_text' && 'text' in contentItem) {
|
|
answer += contentItem.text;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Extract web search queries
|
|
if (item.type === 'web_search_call' && 'action' in item) {
|
|
const searchItem = item as any;
|
|
if (searchItem.action && searchItem.action.type === 'search' && 'query' in searchItem.action) {
|
|
searchQueries.push(searchItem.action.query);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Parse sources from markdown links in the answer
|
|
const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
|
|
let match: RegExpExecArray | null;
|
|
|
|
while ((match = urlRegex.exec(answer)) !== null) {
|
|
sources.push({
|
|
title: match[1],
|
|
url: match[2],
|
|
snippet: ''
|
|
});
|
|
}
|
|
|
|
return {
|
|
answer,
|
|
sources,
|
|
searchQueries: searchQueries.length > 0 ? searchQueries : undefined,
|
|
metadata: {
|
|
model,
|
|
searchDepth: optionsArg.searchDepth || 'basic',
|
|
tokensUsed: result.usage?.total_tokens
|
|
}
|
|
};
|
|
} catch (error) {
|
|
console.error('Research API error:', error);
|
|
throw new Error(`Failed to perform research: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Image generation using OpenAI's gpt-image-1 or DALL-E models
|
|
*/
|
|
public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
|
|
const model = optionsArg.model || this.options.imageModel || 'gpt-image-1';
|
|
|
|
try {
|
|
const requestParams: any = {
|
|
model,
|
|
prompt: optionsArg.prompt,
|
|
n: optionsArg.n || 1,
|
|
};
|
|
|
|
// Add gpt-image-1 specific parameters
|
|
if (model === 'gpt-image-1') {
|
|
if (optionsArg.quality) requestParams.quality = optionsArg.quality;
|
|
if (optionsArg.size) requestParams.size = optionsArg.size;
|
|
if (optionsArg.background) requestParams.background = optionsArg.background;
|
|
if (optionsArg.outputFormat) requestParams.output_format = optionsArg.outputFormat;
|
|
if (optionsArg.outputCompression !== undefined) requestParams.output_compression = optionsArg.outputCompression;
|
|
if (optionsArg.moderation) requestParams.moderation = optionsArg.moderation;
|
|
if (optionsArg.stream !== undefined) requestParams.stream = optionsArg.stream;
|
|
if (optionsArg.partialImages !== undefined) requestParams.partial_images = optionsArg.partialImages;
|
|
} else if (model === 'dall-e-3') {
|
|
// DALL-E 3 specific parameters
|
|
if (optionsArg.quality) requestParams.quality = optionsArg.quality;
|
|
if (optionsArg.size) requestParams.size = optionsArg.size;
|
|
if (optionsArg.style) requestParams.style = optionsArg.style;
|
|
requestParams.response_format = 'b64_json'; // Always use base64 for consistency
|
|
} else if (model === 'dall-e-2') {
|
|
// DALL-E 2 specific parameters
|
|
if (optionsArg.size) requestParams.size = optionsArg.size;
|
|
requestParams.response_format = 'b64_json';
|
|
}
|
|
|
|
const result = await this.openAiApiClient.images.generate(requestParams);
|
|
|
|
const images = (result.data || []).map(img => ({
|
|
b64_json: img.b64_json,
|
|
url: img.url,
|
|
revisedPrompt: img.revised_prompt
|
|
}));
|
|
|
|
return {
|
|
images,
|
|
metadata: {
|
|
model,
|
|
quality: result.quality,
|
|
size: result.size,
|
|
outputFormat: result.output_format,
|
|
tokensUsed: result.usage?.total_tokens
|
|
}
|
|
};
|
|
} catch (error) {
|
|
console.error('Image generation error:', error);
|
|
throw new Error(`Failed to generate image: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Image editing using OpenAI's gpt-image-1 or DALL-E 2 models
|
|
*/
|
|
public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
|
|
const model = optionsArg.model || this.options.imageModel || 'gpt-image-1';
|
|
|
|
try {
|
|
const requestParams: any = {
|
|
model,
|
|
image: optionsArg.image,
|
|
prompt: optionsArg.prompt,
|
|
n: optionsArg.n || 1,
|
|
};
|
|
|
|
// Add mask if provided
|
|
if (optionsArg.mask) {
|
|
requestParams.mask = optionsArg.mask;
|
|
}
|
|
|
|
// Add gpt-image-1 specific parameters
|
|
if (model === 'gpt-image-1') {
|
|
if (optionsArg.quality) requestParams.quality = optionsArg.quality;
|
|
if (optionsArg.size) requestParams.size = optionsArg.size;
|
|
if (optionsArg.background) requestParams.background = optionsArg.background;
|
|
if (optionsArg.outputFormat) requestParams.output_format = optionsArg.outputFormat;
|
|
if (optionsArg.outputCompression !== undefined) requestParams.output_compression = optionsArg.outputCompression;
|
|
if (optionsArg.stream !== undefined) requestParams.stream = optionsArg.stream;
|
|
if (optionsArg.partialImages !== undefined) requestParams.partial_images = optionsArg.partialImages;
|
|
} else if (model === 'dall-e-2') {
|
|
// DALL-E 2 specific parameters
|
|
if (optionsArg.size) requestParams.size = optionsArg.size;
|
|
requestParams.response_format = 'b64_json';
|
|
}
|
|
|
|
const result = await this.openAiApiClient.images.edit(requestParams);
|
|
|
|
const images = (result.data || []).map(img => ({
|
|
b64_json: img.b64_json,
|
|
url: img.url,
|
|
revisedPrompt: img.revised_prompt
|
|
}));
|
|
|
|
return {
|
|
images,
|
|
metadata: {
|
|
model,
|
|
quality: result.quality,
|
|
size: result.size,
|
|
outputFormat: result.output_format,
|
|
tokensUsed: result.usage?.total_tokens
|
|
}
|
|
};
|
|
} catch (error) {
|
|
console.error('Image edit error:', error);
|
|
throw new Error(`Failed to edit image: ${error.message}`);
|
|
}
|
|
}
|
|
} |