fix(core): Fix OpenAI chat streaming and PDF document processing logic.

This commit is contained in:
Philipp Kunz 2025-02-25 18:23:28 +00:00
parent 9b38a3c06e
commit 6ce442354e
5 changed files with 522 additions and 257 deletions

View File

@ -1,5 +1,12 @@
# Changelog
## 2025-02-25 - 0.4.2 - fix(core)
Fix OpenAI chat streaming and PDF document processing logic.
- Updated OpenAI chat streaming to handle new async iterable format.
- Improved PDF document processing by filtering out empty image buffers.
- Removed unsupported temperature options from OpenAI requests.
## 2025-02-25 - 0.4.1 - fix(provider)
Fix provider modules for consistency

View File

@ -27,7 +27,7 @@
"@push.rocks/smartarray": "^1.1.0",
"@push.rocks/smartfile": "^11.2.0",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpdf": "^3.1.8",
"@push.rocks/smartpdf": "^3.2.2",
"@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrequest": "^2.0.23",
"@push.rocks/webstream": "^1.0.10",
@ -66,5 +66,10 @@
"audio responses",
"text-to-speech",
"streaming chat"
],
"pnpm": {
"onlyBuiltDependencies": [
"puppeteer"
]
}
}

679
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartai',
version: '0.4.1',
version: '0.4.2',
description: 'A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.'
}

View File

@ -75,21 +75,23 @@ export class OpenAiProvider extends MultiModalModel {
// If we have a complete message, send it to OpenAI
if (currentMessage) {
const messageToSend = { role: "user" as const, content: currentMessage.content };
const stream = await this.openAiApiClient.chat.completions.create({
model: this.options.chatModel ?? 'o3-mini',
temperature: 0,
const chatModel = this.options.chatModel ?? 'o3-mini';
const requestParams: any = {
model: chatModel,
messages: [messageToSend],
stream: true,
});
};
// Temperature is omitted since the model does not support it.
const stream = await this.openAiApiClient.chat.completions.create(requestParams);
// Explicitly cast the stream as an async iterable to satisfy TypeScript.
const streamAsyncIterable = stream as unknown as AsyncIterableIterator<any>;
// Process each chunk from OpenAI
for await (const chunk of stream) {
for await (const chunk of streamAsyncIterable) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
controller.enqueue(content);
}
}
currentMessage = null;
}
},
@ -119,15 +121,17 @@ export class OpenAiProvider extends MultiModalModel {
content: string;
}[];
}) {
const result = await this.openAiApiClient.chat.completions.create({
model: this.options.chatModel ?? 'o3-mini',
temperature: 0,
const chatModel = this.options.chatModel ?? 'o3-mini';
const requestParams: any = {
model: chatModel,
messages: [
{ role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory,
{ role: 'user', content: optionsArg.userMessage },
],
});
};
// Temperature parameter removed to avoid unsupported error.
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return {
role: result.choices[0].message.role as 'assistant',
message: result.choices[0].message.content,
@ -159,27 +163,30 @@ export class OpenAiProvider extends MultiModalModel {
}) {
let pdfDocumentImageBytesArray: Uint8Array[] = [];
// Convert each PDF into one or more image byte arrays.
const smartpdfInstance = new plugins.smartpdf.SmartPdf();
await smartpdfInstance.start();
for (const pdfDocument of optionsArg.pdfDocuments) {
const documentImageArray = await this.smartpdfInstance.convertPDFToPngBytes(pdfDocument);
const documentImageArray = await smartpdfInstance.convertPDFToPngBytes(pdfDocument);
pdfDocumentImageBytesArray = pdfDocumentImageBytesArray.concat(documentImageArray);
}
await smartpdfInstance.stop();
console.log(`image smartfile array`);
console.log(pdfDocumentImageBytesArray.map((smartfile) => smartfile.length));
const smartfileArray = await plugins.smartarray.map(
pdfDocumentImageBytesArray,
async (pdfDocumentImageBytes) => {
return plugins.smartfile.SmartFile.fromBuffer(
'pdfDocumentImage.jpg',
Buffer.from(pdfDocumentImageBytes)
);
}
);
// Filter out any empty buffers to avoid sending invalid image URLs.
const validImageBytesArray = pdfDocumentImageBytesArray.filter(imageBytes => imageBytes && imageBytes.length > 0);
const imageAttachments = validImageBytesArray.map(imageBytes => ({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
},
}));
const result = await this.openAiApiClient.chat.completions.create({
model: this.options.chatModel ?? 'o3-mini',
temperature: 0,
const chatModel = this.options.chatModel ?? 'gpt-4o';
const requestParams: any = {
model: chatModel,
messages: [
{ role: 'system', content: optionsArg.systemMessage },
...optionsArg.messageHistory,
@ -187,31 +194,22 @@ export class OpenAiProvider extends MultiModalModel {
role: 'user',
content: [
{ type: 'text', text: optionsArg.userMessage },
...(() => {
const returnArray = [];
for (const imageBytes of pdfDocumentImageBytesArray) {
returnArray.push({
type: 'image_url',
image_url: {
url: 'data:image/png;base64,' + Buffer.from(imageBytes).toString('base64'),
},
});
}
return returnArray;
})(),
...imageAttachments,
],
},
],
});
};
// Temperature parameter removed.
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return {
message: result.choices[0].message,
};
}
public async vision(optionsArg: { image: Buffer; prompt: string }): Promise<string> {
const result = await this.openAiApiClient.chat.completions.create({
model: this.options.visionModel ?? 'o3-mini',
temperature: 0,
const visionModel = this.options.visionModel ?? 'gpt-4o';
const requestParams: any = {
model: visionModel,
messages: [
{
role: 'user',
@ -227,8 +225,8 @@ export class OpenAiProvider extends MultiModalModel {
}
],
max_tokens: 300
});
};
const result = await this.openAiApiClient.chat.completions.create(requestParams);
return result.choices[0].message.content || '';
}
}