353 lines
11 KiB
TypeScript
353 lines
11 KiB
TypeScript
|
import * as os from 'node:os';
|
||
|
|
import { Action, type IActionContext } from './base-action.ts';
|
||
|
|
import { logger } from '../logger.ts';
|
||
|
|
import { PROXMOX, UI } from '../constants.ts';
|
||
|
|
|
||
|
|
/**
|
||
|
|
* ProxmoxAction - Gracefully shuts down Proxmox VMs and LXC containers
|
||
|
|
*
|
||
|
|
* Uses the Proxmox REST API via HTTPS with API token authentication.
|
||
|
|
* Shuts down running QEMU VMs and LXC containers, waits for completion,
|
||
|
|
* and optionally force-stops any that don't respond.
|
||
|
|
*
|
||
|
|
* This action should be placed BEFORE shutdown actions in the action chain
|
||
|
|
* so that VMs are stopped before the host is shut down.
|
||
|
|
*/
|
||
|
|
export class ProxmoxAction extends Action {
|
||
|
|
readonly type = 'proxmox';
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Execute the Proxmox shutdown action
|
||
|
|
*/
|
||
|
|
async execute(context: IActionContext): Promise<void> {
|
||
|
|
if (!this.shouldExecute(context)) {
|
||
|
|
logger.info(
|
||
|
|
`Proxmox action skipped (trigger mode: ${
|
||
|
|
this.config.triggerMode || 'powerChangesAndThresholds'
|
||
|
|
})`,
|
||
|
|
);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
const host = this.config.proxmoxHost || PROXMOX.DEFAULT_HOST;
|
||
|
|
const port = this.config.proxmoxPort || PROXMOX.DEFAULT_PORT;
|
||
|
|
const node = this.config.proxmoxNode || os.hostname();
|
||
|
|
const tokenId = this.config.proxmoxTokenId;
|
||
|
|
const tokenSecret = this.config.proxmoxTokenSecret;
|
||
|
|
const excludeIds = new Set(this.config.proxmoxExcludeIds || []);
|
||
|
|
const stopTimeout = (this.config.proxmoxStopTimeout || PROXMOX.DEFAULT_STOP_TIMEOUT_SECONDS) * 1000;
|
||
|
|
const forceStop = this.config.proxmoxForceStop !== false; // default true
|
||
|
|
const insecure = this.config.proxmoxInsecure !== false; // default true
|
||
|
|
|
||
|
|
if (!tokenId || !tokenSecret) {
|
||
|
|
logger.error('Proxmox API token ID and secret are required');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
const baseUrl = `https://${host}:${port}${PROXMOX.API_BASE}`;
|
||
|
|
const headers: Record<string, string> = {
|
||
|
|
'Authorization': `PVEAPIToken=${tokenId}=${tokenSecret}`,
|
||
|
|
};
|
||
|
|
|
||
|
|
logger.log('');
|
||
|
|
logger.logBoxTitle('Proxmox VM Shutdown', UI.WIDE_BOX_WIDTH, 'warning');
|
||
|
|
logger.logBoxLine(`Node: ${node}`);
|
||
|
|
logger.logBoxLine(`API: ${host}:${port}`);
|
||
|
|
logger.logBoxLine(`UPS: ${context.upsName} (${context.powerStatus})`);
|
||
|
|
logger.logBoxLine(`Trigger: ${context.triggerReason}`);
|
||
|
|
if (excludeIds.size > 0) {
|
||
|
|
logger.logBoxLine(`Excluded IDs: ${[...excludeIds].join(', ')}`);
|
||
|
|
}
|
||
|
|
logger.logBoxEnd();
|
||
|
|
logger.log('');
|
||
|
|
|
||
|
|
try {
|
||
|
|
// Collect running VMs and CTs
|
||
|
|
const runningVMs = await this.getRunningVMs(baseUrl, node, headers, insecure);
|
||
|
|
const runningCTs = await this.getRunningCTs(baseUrl, node, headers, insecure);
|
||
|
|
|
||
|
|
// Filter out excluded IDs
|
||
|
|
const vmsToStop = runningVMs.filter((vm) => !excludeIds.has(vm.vmid));
|
||
|
|
const ctsToStop = runningCTs.filter((ct) => !excludeIds.has(ct.vmid));
|
||
|
|
|
||
|
|
const totalToStop = vmsToStop.length + ctsToStop.length;
|
||
|
|
if (totalToStop === 0) {
|
||
|
|
logger.info('No running VMs or containers to shut down');
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
logger.info(`Shutting down ${vmsToStop.length} VMs and ${ctsToStop.length} containers...`);
|
||
|
|
|
||
|
|
// Send shutdown commands to all VMs and CTs
|
||
|
|
for (const vm of vmsToStop) {
|
||
|
|
await this.shutdownVM(baseUrl, node, vm.vmid, headers, insecure);
|
||
|
|
logger.dim(` Shutdown sent to VM ${vm.vmid} (${vm.name || 'unnamed'})`);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (const ct of ctsToStop) {
|
||
|
|
await this.shutdownCT(baseUrl, node, ct.vmid, headers, insecure);
|
||
|
|
logger.dim(` Shutdown sent to CT ${ct.vmid} (${ct.name || 'unnamed'})`);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Poll until all stopped or timeout
|
||
|
|
const allIds = [
|
||
|
|
...vmsToStop.map((vm) => ({ type: 'qemu' as const, vmid: vm.vmid, name: vm.name })),
|
||
|
|
...ctsToStop.map((ct) => ({ type: 'lxc' as const, vmid: ct.vmid, name: ct.name })),
|
||
|
|
];
|
||
|
|
|
||
|
|
const remaining = await this.waitForShutdown(
|
||
|
|
baseUrl,
|
||
|
|
node,
|
||
|
|
allIds,
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
stopTimeout,
|
||
|
|
);
|
||
|
|
|
||
|
|
if (remaining.length > 0 && forceStop) {
|
||
|
|
logger.warn(`${remaining.length} VMs/CTs didn't shut down gracefully, force-stopping...`);
|
||
|
|
for (const item of remaining) {
|
||
|
|
try {
|
||
|
|
if (item.type === 'qemu') {
|
||
|
|
await this.stopVM(baseUrl, node, item.vmid, headers, insecure);
|
||
|
|
} else {
|
||
|
|
await this.stopCT(baseUrl, node, item.vmid, headers, insecure);
|
||
|
|
}
|
||
|
|
logger.dim(` Force-stopped ${item.type} ${item.vmid} (${item.name || 'unnamed'})`);
|
||
|
|
} catch (error) {
|
||
|
|
logger.error(
|
||
|
|
` Failed to force-stop ${item.type} ${item.vmid}: ${
|
||
|
|
error instanceof Error ? error.message : String(error)
|
||
|
|
}`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (remaining.length > 0) {
|
||
|
|
logger.warn(`${remaining.length} VMs/CTs still running (force-stop disabled)`);
|
||
|
|
}
|
||
|
|
|
||
|
|
logger.success('Proxmox shutdown sequence completed');
|
||
|
|
} catch (error) {
|
||
|
|
logger.error(
|
||
|
|
`Proxmox action failed: ${error instanceof Error ? error.message : String(error)}`,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Make an API request to the Proxmox server
|
||
|
|
*/
|
||
|
|
private async apiRequest(
|
||
|
|
url: string,
|
||
|
|
method: string,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<unknown> {
|
||
|
|
const fetchOptions: RequestInit = {
|
||
|
|
method,
|
||
|
|
headers,
|
||
|
|
};
|
||
|
|
|
||
|
|
// Use NODE_TLS_REJECT_UNAUTHORIZED for insecure mode (self-signed certs)
|
||
|
|
if (insecure) {
|
||
|
|
// deno-lint-ignore no-explicit-any
|
||
|
|
(globalThis as any).process?.env && ((globalThis as any).process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0');
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
const response = await fetch(url, fetchOptions);
|
||
|
|
|
||
|
|
if (!response.ok) {
|
||
|
|
const body = await response.text();
|
||
|
|
throw new Error(`Proxmox API error ${response.status}: ${body}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
return await response.json();
|
||
|
|
} finally {
|
||
|
|
// Restore TLS verification
|
||
|
|
if (insecure) {
|
||
|
|
// deno-lint-ignore no-explicit-any
|
||
|
|
(globalThis as any).process?.env && ((globalThis as any).process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Get list of running QEMU VMs
|
||
|
|
*/
|
||
|
|
private async getRunningVMs(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<Array<{ vmid: number; name: string }>> {
|
||
|
|
try {
|
||
|
|
const response = await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/qemu`,
|
||
|
|
'GET',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
) as { data: Array<{ vmid: number; name: string; status: string }> };
|
||
|
|
|
||
|
|
return (response.data || [])
|
||
|
|
.filter((vm) => vm.status === 'running')
|
||
|
|
.map((vm) => ({ vmid: vm.vmid, name: vm.name || '' }));
|
||
|
|
} catch (error) {
|
||
|
|
logger.error(
|
||
|
|
`Failed to list VMs: ${error instanceof Error ? error.message : String(error)}`,
|
||
|
|
);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Get list of running LXC containers
|
||
|
|
*/
|
||
|
|
private async getRunningCTs(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<Array<{ vmid: number; name: string }>> {
|
||
|
|
try {
|
||
|
|
const response = await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/lxc`,
|
||
|
|
'GET',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
) as { data: Array<{ vmid: number; name: string; status: string }> };
|
||
|
|
|
||
|
|
return (response.data || [])
|
||
|
|
.filter((ct) => ct.status === 'running')
|
||
|
|
.map((ct) => ({ vmid: ct.vmid, name: ct.name || '' }));
|
||
|
|
} catch (error) {
|
||
|
|
logger.error(
|
||
|
|
`Failed to list CTs: ${error instanceof Error ? error.message : String(error)}`,
|
||
|
|
);
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Send graceful shutdown to a QEMU VM
|
||
|
|
*/
|
||
|
|
private async shutdownVM(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
vmid: number,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<void> {
|
||
|
|
await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/qemu/${vmid}/status/shutdown`,
|
||
|
|
'POST',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Send graceful shutdown to an LXC container
|
||
|
|
*/
|
||
|
|
private async shutdownCT(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
vmid: number,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<void> {
|
||
|
|
await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/lxc/${vmid}/status/shutdown`,
|
||
|
|
'POST',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Force-stop a QEMU VM
|
||
|
|
*/
|
||
|
|
private async stopVM(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
vmid: number,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<void> {
|
||
|
|
await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/qemu/${vmid}/status/stop`,
|
||
|
|
'POST',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Force-stop an LXC container
|
||
|
|
*/
|
||
|
|
private async stopCT(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
vmid: number,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
): Promise<void> {
|
||
|
|
await this.apiRequest(
|
||
|
|
`${baseUrl}/nodes/${node}/lxc/${vmid}/status/stop`,
|
||
|
|
'POST',
|
||
|
|
headers,
|
||
|
|
insecure,
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Wait for VMs/CTs to shut down, return any that are still running after timeout
|
||
|
|
*/
|
||
|
|
private async waitForShutdown(
|
||
|
|
baseUrl: string,
|
||
|
|
node: string,
|
||
|
|
items: Array<{ type: 'qemu' | 'lxc'; vmid: number; name: string }>,
|
||
|
|
headers: Record<string, string>,
|
||
|
|
insecure: boolean,
|
||
|
|
timeout: number,
|
||
|
|
): Promise<Array<{ type: 'qemu' | 'lxc'; vmid: number; name: string }>> {
|
||
|
|
const startTime = Date.now();
|
||
|
|
let remaining = [...items];
|
||
|
|
|
||
|
|
while (remaining.length > 0 && (Date.now() - startTime) < timeout) {
|
||
|
|
// Wait before polling
|
||
|
|
await new Promise((resolve) => setTimeout(resolve, PROXMOX.STATUS_POLL_INTERVAL_SECONDS * 1000));
|
||
|
|
|
||
|
|
// Check which are still running
|
||
|
|
const stillRunning: typeof remaining = [];
|
||
|
|
|
||
|
|
for (const item of remaining) {
|
||
|
|
try {
|
||
|
|
const statusUrl = `${baseUrl}/nodes/${node}/${item.type}/${item.vmid}/status/current`;
|
||
|
|
const response = await this.apiRequest(statusUrl, 'GET', headers, insecure) as {
|
||
|
|
data: { status: string };
|
||
|
|
};
|
||
|
|
|
||
|
|
if (response.data?.status === 'running') {
|
||
|
|
stillRunning.push(item);
|
||
|
|
} else {
|
||
|
|
logger.dim(` ${item.type} ${item.vmid} (${item.name}) stopped`);
|
||
|
|
}
|
||
|
|
} catch (_error) {
|
||
|
|
// If we can't check status, assume it might still be running
|
||
|
|
stillRunning.push(item);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
remaining = stillRunning;
|
||
|
|
|
||
|
|
if (remaining.length > 0) {
|
||
|
|
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||
|
|
logger.dim(` Waiting... ${remaining.length} still running (${elapsed}s elapsed)`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return remaining;
|
||
|
|
}
|
||
|
|
}
|