modelgrid.com/modelgrid
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat(cluster,api,models,cli): add cluster-aware model catalog deployments and request routing

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-20 23:00:50 +00:00
parent 83cacd0cf1
commit 4f2266e1b7
55 changed files with 3970 additions and 1630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ts/docker/container-runtime.ts
+20 -16
View File
@@ -71,7 +71,11 @@ export class ContainerRuntime {
logger.success(`Started existing container: ${containerName}`);
return true;
} catch (error) {
logger.error(`Failed to start existing container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to start existing container: ${
error instanceof Error ? error.message : String(error)
}`,
);
// Try to remove and recreate
await this.removeContainer(config.id);
}
@@ -93,7 +97,9 @@ export class ContainerRuntime {
return true;
} catch (error) {
logger.error(`Failed to start container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to start container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -118,7 +124,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} stopped`);
return true;
} catch (error) {
logger.error(`Failed to stop container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to stop container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -140,7 +148,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} removed`);
return true;
} catch (error) {
logger.error(`Failed to remove container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to remove container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -164,7 +174,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} restarted`);
return true;
} catch (error) {
logger.error(`Failed to restart container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to restart container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -248,7 +260,9 @@ export class ContainerRuntime {
status.cpuUsage = stats.cpuUsage;
}
} catch (error) {
logger.dim(`Error getting container status: ${error instanceof Error ? error.message : String(error)}`);
logger.dim(
`Error getting container status: ${error instanceof Error ? error.message : String(error)}`,
);
}
return status;
@@ -295,16 +309,6 @@ export class ContainerRuntime {
try {
switch (config.type) {
case 'ollama': {
// Query Ollama API for loaded models
const { stdout } = await execAsync(
`docker exec ${containerName} curl -s http://localhost:11434/api/tags`,
{ timeout: 5000 },
);
const data = JSON.parse(stdout);
return (data.models || []).map((m: { name: string }) => m.name);
}
case 'vllm':
case 'tgi': {
// These typically serve a single model