refactor(timeout): reuse shared api request timeout
This commit is contained in:
@@ -6,6 +6,7 @@ import * as http from 'node:http';
|
||||
import type { IApiError, IChatCompletionRequest } from '../../interfaces/api.ts';
|
||||
import { ClusterCoordinator } from '../../cluster/coordinator.ts';
|
||||
import { ContainerManager } from '../../containers/container-manager.ts';
|
||||
import { API_SERVER } from '../../constants.ts';
|
||||
import { logger } from '../../logger.ts';
|
||||
import { ModelRegistry } from '../../models/registry.ts';
|
||||
import { ModelLoader } from '../../models/loader.ts';
|
||||
@@ -158,11 +159,14 @@ export class ChatHandler {
|
||||
targetEndpoint: string,
|
||||
body: IChatCompletionRequest,
|
||||
): Promise<void> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), API_SERVER.REQUEST_TIMEOUT_MS);
|
||||
const response = await fetch(`${targetEndpoint}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: this.buildForwardHeaders(req),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
signal: controller.signal,
|
||||
}).finally(() => clearTimeout(timeout));
|
||||
|
||||
if (body.stream) {
|
||||
res.writeHead(response.status, {
|
||||
|
||||
Reference in New Issue
Block a user