modelgrid.com/modelgrid
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat(cluster,api,models,cli): add cluster-aware model catalog deployments and request routing

Browse Source
This commit is contained in:
Jürgen Kunz
2026-04-20 23:00:50 +00:00
parent 83cacd0cf1
commit 4f2266e1b7
55 changed files with 3970 additions and 1630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ts/docker/container-runtime.ts
+20 -16
View File
@@ -71,7 +71,11 @@ export class ContainerRuntime {
logger.success(`Started existing container: ${containerName}`);
return true;
} catch (error) {
logger.error(`Failed to start existing container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to start existing container: ${
error instanceof Error ? error.message : String(error)
}`,
);
// Try to remove and recreate
await this.removeContainer(config.id);
}
@@ -93,7 +97,9 @@ export class ContainerRuntime {
return true;
} catch (error) {
logger.error(`Failed to start container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to start container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -118,7 +124,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} stopped`);
return true;
} catch (error) {
logger.error(`Failed to stop container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to stop container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -140,7 +148,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} removed`);
return true;
} catch (error) {
logger.error(`Failed to remove container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to remove container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -164,7 +174,9 @@ export class ContainerRuntime {
logger.success(`Container ${containerName} restarted`);
return true;
} catch (error) {
logger.error(`Failed to restart container: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to restart container: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -248,7 +260,9 @@ export class ContainerRuntime {
status.cpuUsage = stats.cpuUsage;
}
} catch (error) {
logger.dim(`Error getting container status: ${error instanceof Error ? error.message : String(error)}`);
logger.dim(
`Error getting container status: ${error instanceof Error ? error.message : String(error)}`,
);
}
return status;
@@ -295,16 +309,6 @@ export class ContainerRuntime {
try {
switch (config.type) {
case 'ollama': {
// Query Ollama API for loaded models
const { stdout } = await execAsync(
`docker exec ${containerName} curl -s http://localhost:11434/api/tags`,
{ timeout: 5000 },
);
const data = JSON.parse(stdout);
return (data.models || []).map((m: { name: string }) => m.name);
}
case 'vllm':
case 'tgi': {
// These typically serve a single model
ts/docker/docker-manager.ts
+25 -7
View File
@@ -296,7 +296,9 @@ export class DockerManager {
await execAsync('systemctl enable docker');
logger.success('Docker service started and enabled');
} catch (error) {
logger.warn(`Could not start Docker service: ${error instanceof Error ? error.message : String(error)}`);
logger.warn(
`Could not start Docker service: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
@@ -308,7 +310,9 @@ export class DockerManager {
await execAsync('systemctl stop docker');
logger.success('Docker service stopped');
} catch (error) {
logger.warn(`Could not stop Docker service: ${error instanceof Error ? error.message : String(error)}`);
logger.warn(
`Could not stop Docker service: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
@@ -320,7 +324,11 @@ export class DockerManager {
await execAsync('systemctl restart docker');
logger.success('Docker service restarted');
} catch (error) {
logger.warn(`Could not restart Docker service: ${error instanceof Error ? error.message : String(error)}`);
logger.warn(
`Could not restart Docker service: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
@@ -352,7 +360,9 @@ export class DockerManager {
logger.success(`Created Docker network '${this.networkName}'`);
return true;
} catch (error) {
logger.error(`Failed to create network: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to create network: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -372,7 +382,9 @@ export class DockerManager {
logger.success(`Removed Docker network '${this.networkName}'`);
return true;
} catch (error) {
logger.error(`Failed to remove network: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to remove network: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -389,7 +401,9 @@ export class DockerManager {
logger.success(`Pulled image: ${image}`);
return true;
} catch (error) {
logger.error(`Failed to pull image: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to pull image: ${error instanceof Error ? error.message : String(error)}`,
);
return false;
}
}
@@ -454,7 +468,11 @@ export class DockerManager {
logger.info('Log out and log back in for the change to take effect');
return true;
} catch (error) {
logger.error(`Failed to add user to docker group: ${error instanceof Error ? error.message : String(error)}`);
logger.error(
`Failed to add user to docker group: ${
error instanceof Error ? error.message : String(error)
}`,
);
return false;
}
}