feat(daemon): Add UPSD (NUT) protocol support, Proxmox VM shutdown action, pause/resume monitoring, and network-loss/unreachable handling; bump config version to 4.2
This commit is contained in:
352
ts/actions/proxmox-action.ts
Normal file
352
ts/actions/proxmox-action.ts
Normal file
@@ -0,0 +1,352 @@
|
||||
import * as os from 'node:os';
|
||||
import { Action, type IActionContext } from './base-action.ts';
|
||||
import { logger } from '../logger.ts';
|
||||
import { PROXMOX, UI } from '../constants.ts';
|
||||
|
||||
/**
|
||||
* ProxmoxAction - Gracefully shuts down Proxmox VMs and LXC containers
|
||||
*
|
||||
* Uses the Proxmox REST API via HTTPS with API token authentication.
|
||||
* Shuts down running QEMU VMs and LXC containers, waits for completion,
|
||||
* and optionally force-stops any that don't respond.
|
||||
*
|
||||
* This action should be placed BEFORE shutdown actions in the action chain
|
||||
* so that VMs are stopped before the host is shut down.
|
||||
*/
|
||||
export class ProxmoxAction extends Action {
|
||||
readonly type = 'proxmox';
|
||||
|
||||
/**
|
||||
* Execute the Proxmox shutdown action
|
||||
*/
|
||||
async execute(context: IActionContext): Promise<void> {
|
||||
if (!this.shouldExecute(context)) {
|
||||
logger.info(
|
||||
`Proxmox action skipped (trigger mode: ${
|
||||
this.config.triggerMode || 'powerChangesAndThresholds'
|
||||
})`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const host = this.config.proxmoxHost || PROXMOX.DEFAULT_HOST;
|
||||
const port = this.config.proxmoxPort || PROXMOX.DEFAULT_PORT;
|
||||
const node = this.config.proxmoxNode || os.hostname();
|
||||
const tokenId = this.config.proxmoxTokenId;
|
||||
const tokenSecret = this.config.proxmoxTokenSecret;
|
||||
const excludeIds = new Set(this.config.proxmoxExcludeIds || []);
|
||||
const stopTimeout = (this.config.proxmoxStopTimeout || PROXMOX.DEFAULT_STOP_TIMEOUT_SECONDS) * 1000;
|
||||
const forceStop = this.config.proxmoxForceStop !== false; // default true
|
||||
const insecure = this.config.proxmoxInsecure !== false; // default true
|
||||
|
||||
if (!tokenId || !tokenSecret) {
|
||||
logger.error('Proxmox API token ID and secret are required');
|
||||
return;
|
||||
}
|
||||
|
||||
const baseUrl = `https://${host}:${port}${PROXMOX.API_BASE}`;
|
||||
const headers: Record<string, string> = {
|
||||
'Authorization': `PVEAPIToken=${tokenId}=${tokenSecret}`,
|
||||
};
|
||||
|
||||
logger.log('');
|
||||
logger.logBoxTitle('Proxmox VM Shutdown', UI.WIDE_BOX_WIDTH, 'warning');
|
||||
logger.logBoxLine(`Node: ${node}`);
|
||||
logger.logBoxLine(`API: ${host}:${port}`);
|
||||
logger.logBoxLine(`UPS: ${context.upsName} (${context.powerStatus})`);
|
||||
logger.logBoxLine(`Trigger: ${context.triggerReason}`);
|
||||
if (excludeIds.size > 0) {
|
||||
logger.logBoxLine(`Excluded IDs: ${[...excludeIds].join(', ')}`);
|
||||
}
|
||||
logger.logBoxEnd();
|
||||
logger.log('');
|
||||
|
||||
try {
|
||||
// Collect running VMs and CTs
|
||||
const runningVMs = await this.getRunningVMs(baseUrl, node, headers, insecure);
|
||||
const runningCTs = await this.getRunningCTs(baseUrl, node, headers, insecure);
|
||||
|
||||
// Filter out excluded IDs
|
||||
const vmsToStop = runningVMs.filter((vm) => !excludeIds.has(vm.vmid));
|
||||
const ctsToStop = runningCTs.filter((ct) => !excludeIds.has(ct.vmid));
|
||||
|
||||
const totalToStop = vmsToStop.length + ctsToStop.length;
|
||||
if (totalToStop === 0) {
|
||||
logger.info('No running VMs or containers to shut down');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Shutting down ${vmsToStop.length} VMs and ${ctsToStop.length} containers...`);
|
||||
|
||||
// Send shutdown commands to all VMs and CTs
|
||||
for (const vm of vmsToStop) {
|
||||
await this.shutdownVM(baseUrl, node, vm.vmid, headers, insecure);
|
||||
logger.dim(` Shutdown sent to VM ${vm.vmid} (${vm.name || 'unnamed'})`);
|
||||
}
|
||||
|
||||
for (const ct of ctsToStop) {
|
||||
await this.shutdownCT(baseUrl, node, ct.vmid, headers, insecure);
|
||||
logger.dim(` Shutdown sent to CT ${ct.vmid} (${ct.name || 'unnamed'})`);
|
||||
}
|
||||
|
||||
// Poll until all stopped or timeout
|
||||
const allIds = [
|
||||
...vmsToStop.map((vm) => ({ type: 'qemu' as const, vmid: vm.vmid, name: vm.name })),
|
||||
...ctsToStop.map((ct) => ({ type: 'lxc' as const, vmid: ct.vmid, name: ct.name })),
|
||||
];
|
||||
|
||||
const remaining = await this.waitForShutdown(
|
||||
baseUrl,
|
||||
node,
|
||||
allIds,
|
||||
headers,
|
||||
insecure,
|
||||
stopTimeout,
|
||||
);
|
||||
|
||||
if (remaining.length > 0 && forceStop) {
|
||||
logger.warn(`${remaining.length} VMs/CTs didn't shut down gracefully, force-stopping...`);
|
||||
for (const item of remaining) {
|
||||
try {
|
||||
if (item.type === 'qemu') {
|
||||
await this.stopVM(baseUrl, node, item.vmid, headers, insecure);
|
||||
} else {
|
||||
await this.stopCT(baseUrl, node, item.vmid, headers, insecure);
|
||||
}
|
||||
logger.dim(` Force-stopped ${item.type} ${item.vmid} (${item.name || 'unnamed'})`);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
` Failed to force-stop ${item.type} ${item.vmid}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if (remaining.length > 0) {
|
||||
logger.warn(`${remaining.length} VMs/CTs still running (force-stop disabled)`);
|
||||
}
|
||||
|
||||
logger.success('Proxmox shutdown sequence completed');
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Proxmox action failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an API request to the Proxmox server
|
||||
*/
|
||||
private async apiRequest(
|
||||
url: string,
|
||||
method: string,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<unknown> {
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers,
|
||||
};
|
||||
|
||||
// Use NODE_TLS_REJECT_UNAUTHORIZED for insecure mode (self-signed certs)
|
||||
if (insecure) {
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(globalThis as any).process?.env && ((globalThis as any).process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0');
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, fetchOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text();
|
||||
throw new Error(`Proxmox API error ${response.status}: ${body}`);
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
} finally {
|
||||
// Restore TLS verification
|
||||
if (insecure) {
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(globalThis as any).process?.env && ((globalThis as any).process.env.NODE_TLS_REJECT_UNAUTHORIZED = '1');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of running QEMU VMs
|
||||
*/
|
||||
private async getRunningVMs(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<Array<{ vmid: number; name: string }>> {
|
||||
try {
|
||||
const response = await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/qemu`,
|
||||
'GET',
|
||||
headers,
|
||||
insecure,
|
||||
) as { data: Array<{ vmid: number; name: string; status: string }> };
|
||||
|
||||
return (response.data || [])
|
||||
.filter((vm) => vm.status === 'running')
|
||||
.map((vm) => ({ vmid: vm.vmid, name: vm.name || '' }));
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to list VMs: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of running LXC containers
|
||||
*/
|
||||
private async getRunningCTs(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<Array<{ vmid: number; name: string }>> {
|
||||
try {
|
||||
const response = await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/lxc`,
|
||||
'GET',
|
||||
headers,
|
||||
insecure,
|
||||
) as { data: Array<{ vmid: number; name: string; status: string }> };
|
||||
|
||||
return (response.data || [])
|
||||
.filter((ct) => ct.status === 'running')
|
||||
.map((ct) => ({ vmid: ct.vmid, name: ct.name || '' }));
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to list CTs: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send graceful shutdown to a QEMU VM
|
||||
*/
|
||||
private async shutdownVM(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
vmid: number,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<void> {
|
||||
await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/qemu/${vmid}/status/shutdown`,
|
||||
'POST',
|
||||
headers,
|
||||
insecure,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send graceful shutdown to an LXC container
|
||||
*/
|
||||
private async shutdownCT(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
vmid: number,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<void> {
|
||||
await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/lxc/${vmid}/status/shutdown`,
|
||||
'POST',
|
||||
headers,
|
||||
insecure,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Force-stop a QEMU VM
|
||||
*/
|
||||
private async stopVM(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
vmid: number,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<void> {
|
||||
await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/qemu/${vmid}/status/stop`,
|
||||
'POST',
|
||||
headers,
|
||||
insecure,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Force-stop an LXC container
|
||||
*/
|
||||
private async stopCT(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
vmid: number,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
): Promise<void> {
|
||||
await this.apiRequest(
|
||||
`${baseUrl}/nodes/${node}/lxc/${vmid}/status/stop`,
|
||||
'POST',
|
||||
headers,
|
||||
insecure,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for VMs/CTs to shut down, return any that are still running after timeout
|
||||
*/
|
||||
private async waitForShutdown(
|
||||
baseUrl: string,
|
||||
node: string,
|
||||
items: Array<{ type: 'qemu' | 'lxc'; vmid: number; name: string }>,
|
||||
headers: Record<string, string>,
|
||||
insecure: boolean,
|
||||
timeout: number,
|
||||
): Promise<Array<{ type: 'qemu' | 'lxc'; vmid: number; name: string }>> {
|
||||
const startTime = Date.now();
|
||||
let remaining = [...items];
|
||||
|
||||
while (remaining.length > 0 && (Date.now() - startTime) < timeout) {
|
||||
// Wait before polling
|
||||
await new Promise((resolve) => setTimeout(resolve, PROXMOX.STATUS_POLL_INTERVAL_SECONDS * 1000));
|
||||
|
||||
// Check which are still running
|
||||
const stillRunning: typeof remaining = [];
|
||||
|
||||
for (const item of remaining) {
|
||||
try {
|
||||
const statusUrl = `${baseUrl}/nodes/${node}/${item.type}/${item.vmid}/status/current`;
|
||||
const response = await this.apiRequest(statusUrl, 'GET', headers, insecure) as {
|
||||
data: { status: string };
|
||||
};
|
||||
|
||||
if (response.data?.status === 'running') {
|
||||
stillRunning.push(item);
|
||||
} else {
|
||||
logger.dim(` ${item.type} ${item.vmid} (${item.name}) stopped`);
|
||||
}
|
||||
} catch (_error) {
|
||||
// If we can't check status, assume it might still be running
|
||||
stillRunning.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
remaining = stillRunning;
|
||||
|
||||
if (remaining.length > 0) {
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
logger.dim(` Waiting... ${remaining.length} still running (${elapsed}s elapsed)`);
|
||||
}
|
||||
}
|
||||
|
||||
return remaining;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user