fix(core): Enhanced chat streaming and error handling across providers
This commit is contained in:
@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartai',
|
||||
version: '0.0.18',
|
||||
version: '0.0.19',
|
||||
description: 'A TypeScript library for integrating and interacting with multiple AI models, offering capabilities for chat and potentially audio responses.'
|
||||
}
|
||||
|
||||
@ -1,32 +1,65 @@
|
||||
/**
|
||||
* Message format for chat interactions
|
||||
*/
|
||||
export interface ChatMessage {
|
||||
role: 'assistant' | 'user' | 'system';
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for chat interactions
|
||||
*/
|
||||
export interface ChatOptions {
|
||||
systemMessage: string;
|
||||
userMessage: string;
|
||||
messageHistory: ChatMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Response format for chat interactions
|
||||
*/
|
||||
export interface ChatResponse {
|
||||
role: 'assistant';
|
||||
message: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstract base class for multi-modal AI models.
|
||||
* Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
|
||||
*/
|
||||
export abstract class MultiModalModel {
|
||||
/**
|
||||
* starts the model
|
||||
* Initializes the model and any necessary resources
|
||||
* Should be called before using any other methods
|
||||
*/
|
||||
abstract start(): Promise<void>;
|
||||
|
||||
/**
|
||||
* stops the model
|
||||
* Cleans up any resources used by the model
|
||||
* Should be called when the model is no longer needed
|
||||
*/
|
||||
abstract stop(): Promise<void>;
|
||||
|
||||
public abstract chat(optionsArg: {
|
||||
systemMessage: string,
|
||||
userMessage: string,
|
||||
messageHistory: {
|
||||
role: 'assistant' | 'user';
|
||||
content: string;
|
||||
}[]
|
||||
}): Promise<{
|
||||
role: 'assistant';
|
||||
message: string;
|
||||
}>
|
||||
/**
|
||||
* Synchronous chat interaction with the model
|
||||
* @param optionsArg Options containing system message, user message, and message history
|
||||
* @returns Promise resolving to the assistant's response
|
||||
*/
|
||||
public abstract chat(optionsArg: ChatOptions): Promise<ChatResponse>;
|
||||
|
||||
/**
|
||||
* Defines a streaming interface for chat interactions.
|
||||
* The implementation will vary based on the specific AI model.
|
||||
* @param input
|
||||
* Streaming interface for chat interactions
|
||||
* Allows for real-time responses from the model
|
||||
* @param input Stream of user messages
|
||||
* @returns Stream of model responses
|
||||
*/
|
||||
public abstract chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>>;
|
||||
|
||||
public abstract chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>>;
|
||||
|
||||
/**
|
||||
* Text-to-speech conversion
|
||||
* @param optionsArg Options containing the message to convert to speech
|
||||
* @returns Promise resolving to a readable stream of audio data
|
||||
* @throws Error if the provider doesn't support audio generation
|
||||
*/
|
||||
public abstract audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream>;
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
||||
import type { ChatOptions, ChatResponse, ChatMessage } from './abstract.classes.multimodal.js';
|
||||
|
||||
export interface IAnthropicProviderOptions {
|
||||
anthropicToken: string;
|
||||
@ -23,39 +24,109 @@ export class AnthropicProvider extends MultiModalModel {
|
||||
|
||||
async stop() {}
|
||||
|
||||
public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
|
||||
// TODO: implement for OpenAI
|
||||
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
||||
// Create a TextDecoder to handle incoming chunks
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let currentMessage: { role: string; content: string; } | null = null;
|
||||
|
||||
const returnStream = new ReadableStream();
|
||||
return returnStream;
|
||||
// Create a TransformStream to process the input
|
||||
const transform = new TransformStream<Uint8Array, string>({
|
||||
async transform(chunk, controller) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
|
||||
// Try to parse complete JSON messages from the buffer
|
||||
while (true) {
|
||||
const newlineIndex = buffer.indexOf('\n');
|
||||
if (newlineIndex === -1) break;
|
||||
|
||||
const line = buffer.slice(0, newlineIndex);
|
||||
buffer = buffer.slice(newlineIndex + 1);
|
||||
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
currentMessage = {
|
||||
role: message.role || 'user',
|
||||
content: message.content || '',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to parse message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have a complete message, send it to Anthropic
|
||||
if (currentMessage) {
|
||||
const stream = await this.anthropicApiClient.messages.create({
|
||||
model: 'claude-3-opus-20240229',
|
||||
messages: [{ role: currentMessage.role, content: currentMessage.content }],
|
||||
system: '',
|
||||
stream: true,
|
||||
max_tokens: 4000,
|
||||
});
|
||||
|
||||
// Process each chunk from Anthropic
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.delta?.text;
|
||||
if (content) {
|
||||
controller.enqueue(content);
|
||||
}
|
||||
}
|
||||
|
||||
currentMessage = null;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
if (buffer) {
|
||||
try {
|
||||
const message = JSON.parse(buffer);
|
||||
controller.enqueue(message.content || '');
|
||||
} catch (e) {
|
||||
console.error('Failed to parse remaining buffer:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the input to our transform stream
|
||||
return input.pipeThrough(transform);
|
||||
}
|
||||
|
||||
// Implementing the synchronous chat interaction
|
||||
public async chat(optionsArg: {
|
||||
systemMessage: string;
|
||||
userMessage: string;
|
||||
messageHistory: {
|
||||
role: 'assistant' | 'user';
|
||||
content: string;
|
||||
}[];
|
||||
}) {
|
||||
public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
|
||||
// Convert message history to Anthropic format
|
||||
const messages = optionsArg.messageHistory.map(msg => ({
|
||||
role: msg.role === 'assistant' ? 'assistant' as const : 'user' as const,
|
||||
content: msg.content
|
||||
}));
|
||||
|
||||
const result = await this.anthropicApiClient.messages.create({
|
||||
model: 'claude-3-opus-20240229',
|
||||
system: optionsArg.systemMessage,
|
||||
messages: [
|
||||
...optionsArg.messageHistory,
|
||||
{ role: 'user', content: optionsArg.userMessage },
|
||||
...messages,
|
||||
{ role: 'user' as const, content: optionsArg.userMessage }
|
||||
],
|
||||
max_tokens: 4000,
|
||||
});
|
||||
|
||||
// Extract text content from the response
|
||||
let message = '';
|
||||
for (const block of result.content) {
|
||||
if ('text' in block) {
|
||||
message += block.text;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
role: result.role as 'assistant',
|
||||
message: result.content.join('\n'),
|
||||
role: 'assistant' as const,
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
private async audio(messageArg: string) {
|
||||
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
||||
// Anthropic does not provide an audio API, so this method is not implemented.
|
||||
throw new Error('Audio generation is not yet supported by Anthropic.');
|
||||
}
|
||||
|
||||
@ -0,0 +1,179 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
||||
import type { ChatOptions, ChatResponse, ChatMessage } from './abstract.classes.multimodal.js';
|
||||
|
||||
export interface IGroqProviderOptions {
|
||||
groqToken: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
export class GroqProvider extends MultiModalModel {
|
||||
private options: IGroqProviderOptions;
|
||||
private baseUrl = 'https://api.groq.com/v1';
|
||||
|
||||
constructor(optionsArg: IGroqProviderOptions) {
|
||||
super();
|
||||
this.options = {
|
||||
...optionsArg,
|
||||
model: optionsArg.model || 'llama-3.3-70b-versatile', // Default model
|
||||
};
|
||||
}
|
||||
|
||||
async start() {}
|
||||
|
||||
async stop() {}
|
||||
|
||||
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
||||
// Create a TextDecoder to handle incoming chunks
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let currentMessage: { role: string; content: string; } | null = null;
|
||||
|
||||
// Create a TransformStream to process the input
|
||||
const transform = new TransformStream<Uint8Array, string>({
|
||||
async transform(chunk, controller) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
|
||||
// Try to parse complete JSON messages from the buffer
|
||||
while (true) {
|
||||
const newlineIndex = buffer.indexOf('\n');
|
||||
if (newlineIndex === -1) break;
|
||||
|
||||
const line = buffer.slice(0, newlineIndex);
|
||||
buffer = buffer.slice(newlineIndex + 1);
|
||||
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
currentMessage = {
|
||||
role: message.role || 'user',
|
||||
content: message.content || '',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to parse message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have a complete message, send it to Groq
|
||||
if (currentMessage) {
|
||||
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.options.groqToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.options.model,
|
||||
messages: [{ role: currentMessage.role, content: currentMessage.content }],
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
// Process each chunk from Groq
|
||||
const reader = response.body?.getReader();
|
||||
if (reader) {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = new TextDecoder().decode(value);
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ')) {
|
||||
const data = line.slice(6);
|
||||
if (data === '[DONE]') break;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
const content = parsed.choices[0]?.delta?.content;
|
||||
if (content) {
|
||||
controller.enqueue(content);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to parse SSE data:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
currentMessage = null;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
if (buffer) {
|
||||
try {
|
||||
const message = JSON.parse(buffer);
|
||||
controller.enqueue(message.content || '');
|
||||
} catch (e) {
|
||||
console.error('Failed to parse remaining buffer:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the input to our transform stream
|
||||
return input.pipeThrough(transform);
|
||||
}
|
||||
|
||||
// Implementing the synchronous chat interaction
|
||||
public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
|
||||
const messages = [
|
||||
// System message
|
||||
{
|
||||
role: 'system',
|
||||
content: optionsArg.systemMessage,
|
||||
},
|
||||
// Message history
|
||||
...optionsArg.messageHistory.map(msg => ({
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
})),
|
||||
// User message
|
||||
{
|
||||
role: 'user',
|
||||
content: optionsArg.userMessage,
|
||||
},
|
||||
];
|
||||
|
||||
const response = await fetch(`${this.baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.options.groqToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.options.model,
|
||||
messages,
|
||||
temperature: 0.7,
|
||||
max_completion_tokens: 1024,
|
||||
stream: false,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.json();
|
||||
throw new Error(`Groq API error: ${error.message || response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
return {
|
||||
role: 'assistant',
|
||||
message: result.choices[0].message.content,
|
||||
};
|
||||
}
|
||||
|
||||
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
||||
// Groq does not provide an audio API, so this method is not implemented.
|
||||
throw new Error('Audio generation is not yet supported by Groq.');
|
||||
}
|
||||
}
|
||||
@ -1,3 +1,170 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
||||
import type { ChatOptions, ChatResponse, ChatMessage } from './abstract.classes.multimodal.js';
|
||||
|
||||
export class OllamaProvider {}
|
||||
export interface IOllamaProviderOptions {
|
||||
baseUrl?: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
export class OllamaProvider extends MultiModalModel {
|
||||
private options: IOllamaProviderOptions;
|
||||
private baseUrl: string;
|
||||
private model: string;
|
||||
|
||||
constructor(optionsArg: IOllamaProviderOptions = {}) {
|
||||
super();
|
||||
this.options = optionsArg;
|
||||
this.baseUrl = optionsArg.baseUrl || 'http://localhost:11434';
|
||||
this.model = optionsArg.model || 'llama2';
|
||||
}
|
||||
|
||||
async start() {
|
||||
// Verify Ollama is running
|
||||
try {
|
||||
const response = await fetch(`${this.baseUrl}/api/tags`);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to connect to Ollama server');
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to connect to Ollama server at ${this.baseUrl}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async stop() {}
|
||||
|
||||
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
||||
// Create a TextDecoder to handle incoming chunks
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let currentMessage: { role: string; content: string; } | null = null;
|
||||
|
||||
// Create a TransformStream to process the input
|
||||
const transform = new TransformStream<Uint8Array, string>({
|
||||
async transform(chunk, controller) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
|
||||
// Try to parse complete JSON messages from the buffer
|
||||
while (true) {
|
||||
const newlineIndex = buffer.indexOf('\n');
|
||||
if (newlineIndex === -1) break;
|
||||
|
||||
const line = buffer.slice(0, newlineIndex);
|
||||
buffer = buffer.slice(newlineIndex + 1);
|
||||
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
currentMessage = {
|
||||
role: message.role || 'user',
|
||||
content: message.content || '',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to parse message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have a complete message, send it to Ollama
|
||||
if (currentMessage) {
|
||||
const response = await fetch(`${this.baseUrl}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
messages: [{ role: currentMessage.role, content: currentMessage.content }],
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
// Process each chunk from Ollama
|
||||
const reader = response.body?.getReader();
|
||||
if (reader) {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = new TextDecoder().decode(value);
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
const content = parsed.message?.content;
|
||||
if (content) {
|
||||
controller.enqueue(content);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to parse Ollama response:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
currentMessage = null;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
if (buffer) {
|
||||
try {
|
||||
const message = JSON.parse(buffer);
|
||||
controller.enqueue(message.content || '');
|
||||
} catch (e) {
|
||||
console.error('Failed to parse remaining buffer:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the input to our transform stream
|
||||
return input.pipeThrough(transform);
|
||||
}
|
||||
|
||||
// Implementing the synchronous chat interaction
|
||||
public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
|
||||
// Format messages for Ollama
|
||||
const messages = [
|
||||
{ role: 'system', content: optionsArg.systemMessage },
|
||||
...optionsArg.messageHistory,
|
||||
{ role: 'user', content: optionsArg.userMessage }
|
||||
];
|
||||
|
||||
// Make API call to Ollama
|
||||
const response = await fetch(`${this.baseUrl}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.model,
|
||||
messages: messages,
|
||||
stream: false
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama API error: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
return {
|
||||
role: 'assistant' as const,
|
||||
message: result.message.content,
|
||||
};
|
||||
}
|
||||
|
||||
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
||||
throw new Error('Audio generation is not supported by Ollama.');
|
||||
}
|
||||
}
|
||||
@ -27,11 +27,72 @@ export class OpenAiProvider extends MultiModalModel {
|
||||
|
||||
public async stop() {}
|
||||
|
||||
public async chatStream(input: ReadableStream<string>): Promise<ReadableStream<string>> {
|
||||
// TODO: implement for OpenAI
|
||||
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
||||
// Create a TextDecoder to handle incoming chunks
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let currentMessage: { role: string; content: string; } | null = null;
|
||||
|
||||
const returnStream = new ReadableStream();
|
||||
return returnStream;
|
||||
// Create a TransformStream to process the input
|
||||
const transform = new TransformStream<Uint8Array, string>({
|
||||
async transform(chunk, controller) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
|
||||
// Try to parse complete JSON messages from the buffer
|
||||
while (true) {
|
||||
const newlineIndex = buffer.indexOf('\n');
|
||||
if (newlineIndex === -1) break;
|
||||
|
||||
const line = buffer.slice(0, newlineIndex);
|
||||
buffer = buffer.slice(newlineIndex + 1);
|
||||
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
currentMessage = {
|
||||
role: message.role || 'user',
|
||||
content: message.content || '',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to parse message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have a complete message, send it to OpenAI
|
||||
if (currentMessage) {
|
||||
const stream = await this.openAiApiClient.chat.completions.create({
|
||||
model: 'gpt-4',
|
||||
messages: [{ role: currentMessage.role, content: currentMessage.content }],
|
||||
stream: true,
|
||||
});
|
||||
|
||||
// Process each chunk from OpenAI
|
||||
for await (const chunk of stream) {
|
||||
const content = chunk.choices[0]?.delta?.content;
|
||||
if (content) {
|
||||
controller.enqueue(content);
|
||||
}
|
||||
}
|
||||
|
||||
currentMessage = null;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
if (buffer) {
|
||||
try {
|
||||
const message = JSON.parse(buffer);
|
||||
controller.enqueue(message.content || '');
|
||||
} catch (e) {
|
||||
console.error('Failed to parse remaining buffer:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the input to our transform stream
|
||||
return input.pipeThrough(transform);
|
||||
}
|
||||
|
||||
// Implementing the synchronous chat interaction
|
||||
|
||||
@ -1,3 +1,158 @@
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
import { MultiModalModel } from './abstract.classes.multimodal.js';
|
||||
import type { ChatOptions, ChatResponse, ChatMessage } from './abstract.classes.multimodal.js';
|
||||
|
||||
export class PerplexityProvider {}
|
||||
export interface IPerplexityProviderOptions {
|
||||
perplexityToken: string;
|
||||
}
|
||||
|
||||
export class PerplexityProvider extends MultiModalModel {
|
||||
private options: IPerplexityProviderOptions;
|
||||
|
||||
constructor(optionsArg: IPerplexityProviderOptions) {
|
||||
super();
|
||||
this.options = optionsArg;
|
||||
}
|
||||
|
||||
async start() {
|
||||
// Initialize any necessary clients or resources
|
||||
}
|
||||
|
||||
async stop() {}
|
||||
|
||||
public async chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>> {
|
||||
// Create a TextDecoder to handle incoming chunks
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let currentMessage: { role: string; content: string; } | null = null;
|
||||
|
||||
// Create a TransformStream to process the input
|
||||
const transform = new TransformStream<Uint8Array, string>({
|
||||
async transform(chunk, controller) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
|
||||
// Try to parse complete JSON messages from the buffer
|
||||
while (true) {
|
||||
const newlineIndex = buffer.indexOf('\n');
|
||||
if (newlineIndex === -1) break;
|
||||
|
||||
const line = buffer.slice(0, newlineIndex);
|
||||
buffer = buffer.slice(newlineIndex + 1);
|
||||
|
||||
if (line.trim()) {
|
||||
try {
|
||||
const message = JSON.parse(line);
|
||||
currentMessage = {
|
||||
role: message.role || 'user',
|
||||
content: message.content || '',
|
||||
};
|
||||
} catch (e) {
|
||||
console.error('Failed to parse message:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we have a complete message, send it to Perplexity
|
||||
if (currentMessage) {
|
||||
const response = await fetch('https://api.perplexity.ai/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.options.perplexityToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'mixtral-8x7b-instruct',
|
||||
messages: [{ role: currentMessage.role, content: currentMessage.content }],
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
// Process each chunk from Perplexity
|
||||
const reader = response.body?.getReader();
|
||||
if (reader) {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = new TextDecoder().decode(value);
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ')) {
|
||||
const data = line.slice(6);
|
||||
if (data === '[DONE]') break;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
const content = parsed.choices[0]?.delta?.content;
|
||||
if (content) {
|
||||
controller.enqueue(content);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to parse SSE data:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
currentMessage = null;
|
||||
}
|
||||
},
|
||||
|
||||
flush(controller) {
|
||||
if (buffer) {
|
||||
try {
|
||||
const message = JSON.parse(buffer);
|
||||
controller.enqueue(message.content || '');
|
||||
} catch (e) {
|
||||
console.error('Failed to parse remaining buffer:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Connect the input to our transform stream
|
||||
return input.pipeThrough(transform);
|
||||
}
|
||||
|
||||
// Implementing the synchronous chat interaction
|
||||
public async chat(optionsArg: ChatOptions): Promise<ChatResponse> {
|
||||
// Make API call to Perplexity
|
||||
const response = await fetch('https://api.perplexity.ai/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.options.perplexityToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'mixtral-8x7b-instruct', // Using Mixtral model
|
||||
messages: [
|
||||
{ role: 'system', content: optionsArg.systemMessage },
|
||||
...optionsArg.messageHistory,
|
||||
{ role: 'user', content: optionsArg.userMessage }
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Perplexity API error: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
return {
|
||||
role: 'assistant' as const,
|
||||
message: result.choices[0].message.content,
|
||||
};
|
||||
}
|
||||
|
||||
public async audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream> {
|
||||
throw new Error('Audio generation is not supported by Perplexity.');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user