fix(core): Fix OpenAI chat streaming and PDF document processing logic.

This commit is contained in:
Philipp Kunz 2025-02-25 18:23:28 +00:00
parent 9b38a3c06e
commit 6ce442354e
5 changed files with 522 additions and 257 deletions

View File

@ -1,5 +1,12 @@
# Changelog # Changelog
## 2025-02-25 - 0.4.2 - fix(core)
Fix OpenAI chat streaming and PDF document processing logic.
- Updated OpenAI chat streaming to handle new async iterable format.
- Improved PDF document processing by filtering out empty image buffers.
- Removed unsupported temperature options from OpenAI requests.
## 2025-02-25 - 0.4.1 - fix(provider) ## 2025-02-25 - 0.4.1 - fix(provider)
Fix provider modules for consistency Fix provider modules for consistency

View File

@ -27,7 +27,7 @@
"@push.rocks/smartarray": "^1.1.0", "@push.rocks/smartarray": "^1.1.0",
"@push.rocks/smartfile": "^11.2.0", "@push.rocks/smartfile": "^11.2.0",
"@push.rocks/smartpath": "^5.0.18", "@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpdf": "^3.1.8", "@push.rocks/smartpdf": "^3.2.2",
"@push.rocks/smartpromise": "^4.2.3", "@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^2.0.23", "@push.rocks/smartrequest": "^2.0.23",
"@push.rocks/webstream": "^1.0.10", "@push.rocks/webstream": "^1.0.10",
@ -66,5 +66,10 @@
"audio responses", "audio responses",
"text-to-speech", "text-to-speech",
"streaming chat" "streaming chat"
],
"pnpm": {
"onlyBuiltDependencies": [
"puppeteer"
] ]
}
} }

679
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartai', name: '@push.rocks/smartai',
version: '0.4.1', version: '0.4.2',
description: 'A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.' description: 'A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.'
} }

View File

@ -75,21 +75,23 @@ export class OpenAiProvider extends MultiModalModel {
// If we have a complete message, send it to OpenAI // If we have a complete message, send it to OpenAI
if (currentMessage) { if (currentMessage) {
const messageToSend = { role: "user" as const, content: currentMessage.content }; const messageToSend = { role: "user" as const, content: currentMessage.content };
const stream = await this.openAiApiClient.chat.completions.create({ const chatModel = this.options.chatModel ?? 'o3-mini';
model: this.options.chatModel ?? 'o3-mini', const requestParams: any = {
temperature: 0, model: chatModel,
messages: [messageToSend], messages: [messageToSend],
stream: true, stream: true,
}); };
// Temperature is omitted since the model does not support it.
const stream = await this.openAiApiClient.chat.completions.create(requestParams);
// Explicitly cast the stream as an async iterable to satisfy TypeScript.
const streamAsyncIterable = stream as unknown as AsyncIterableIterator<any>;
// Process each chunk from OpenAI // Process each chunk from OpenAI
for await (const chunk of stream) { for await (const chunk of streamAsyncIterable) {
const content = chunk.choices[0]?.delta?.content; const content = chunk.choices[0]?.delta?.content;
if (content) { if (content) {
controller.enqueue(content); controller.enqueue(content);
} }
} }
currentMessage = null; currentMessage = null;
} }
}, },
@ -119,15 +121,17 @@ export class OpenAiProvider extends MultiModalModel {
content: string; content: string;
}[]; }[];
}) { }) {
const result = await this.openAiApiClient.chat.completions.create({ const chatModel = this.options.chatModel ?? 'o3-mini';
model: this.options.chatModel ?? 'o3-mini', const requestParams: any = {
temperature: 0, model: chatModel,
messages: [ messages: [
{ role: 'system', content: optionsArg.systemMessage }, { role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory, ...optionsArg.messageHistory,
{ role: 'user', content: optionsArg.userMessage }, { role: 'user', content: optionsArg.userMessage },
], ],
}); };
// Temperature parameter removed to avoid unsupported error.
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return { return {
role: result.choices[0].message.role as 'assistant', role: result.choices[0].message.role as 'assistant',
message: result.choices[0].message.content, message: result.choices[0].message.content,
@ -159,27 +163,30 @@ export class OpenAiProvider extends MultiModalModel {
}) { }) {
let pdfDocumentImageBytesArray: Uint8Array[] = []; let pdfDocumentImageBytesArray: Uint8Array[] = [];
// Convert each PDF into one or more image byte arrays.
const smartpdfInstance = new plugins.smartpdf.SmartPdf();
await smartpdfInstance.start();
for (const pdfDocument of optionsArg.pdfDocuments) { for (const pdfDocument of optionsArg.pdfDocuments) {
const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument); const documentImageArray = await smartpdfInstance.convertPDFToPngBytes(pdfDocument);
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray); pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
} }
await smartpdfInstance.stop();
console.log(`image smartfile array`); console.log(`image smartfile array`);
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length)); console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
const smartfileArray = await plugins.smartarray.map( // Filter out any empty buffers to avoid sending invalid image URLs.
pdfDocumentImageBytesArray, const validImageBytesArray = pdfDocumentImageBytesArray.filter(imageBytes => imageBytes && imageBytes.length > 0);
async (pdfDocumentImageBytes) => { const imageAttachments = validImageBytesArray.map(imageBytes => ({
return plugins.smartfile.SmartFile.fromBuffer( type: 'image_url',
'pdfDocumentImage.jpg', image_url: {
Buffer.from(pdfDocumentImageBytes) url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
); },
} }));
);
const result = await this.openAiApiClient.chat.completions.create({ const chatModel = this.options.chatModel ?? 'gpt-4o';
model: this.options.chatModel ?? 'o3-mini', const requestParams: any = {
temperature: 0, model: chatModel,
messages: [ messages: [
{ role: 'system', content: optionsArg.systemMessage }, { role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory, ...optionsArg.messageHistory,
@ -187,31 +194,22 @@ export class OpenAiProvider extends MultiModalModel {
role: 'user', role: 'user',
content: [ content: [
{ type: 'text', text: optionsArg.userMessage }, { type: 'text', text: optionsArg.userMessage },
...(() => { ...imageAttachments,
const returnArray = [];
for (const imageBytes of pdfDocumentImageBytesArray) {
returnArray.push({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
},
});
}
return returnArray;
})(),
], ],
}, },
], ],
}); };
// Temperature parameter removed.
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return { return {
message: result.choices[0].message, message: result.choices[0].message,
}; };
} }
public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> { public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
const result = await this.openAiApiClient.chat.completions.create({ const visionModel = this.options.visionModel ?? 'gpt-4o';
model: this.options.visionModel ?? 'o3-mini', const requestParams: any = {
temperature: 0, model: visionModel,
messages: [ messages: [
{ {
role: 'user', role: 'user',
@ -227,8 +225,8 @@ export class OpenAiProvider extends MultiModalModel {
} }
], ],
max_tokens: 300 max_tokens: 300
}); };
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return result.choices[0].message.content || ''; return result.choices[0].message.content || '';
} }
} }