- Add process.stdin.destroy() after rl.close() in all interactive commands to properly release stdin and allow process to exit cleanly - Replace raw console.log with logger methods throughout CLI handlers - Convert manual box drawing to logger.logBox() in daemon.ts - Standardize menu formatting with logger.info() and logger.dim() - Improve migration output to only show when migrations actually run Fixes issue where process would not exit after "Setup complete!" message due to stdin keeping the event loop alive.
1025 lines
32 KiB
TypeScript
1025 lines
32 KiB
TypeScript
import process from 'node:process';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { exec, execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { NupstSnmp } from './snmp/manager.ts';
|
|
import type { ISnmpConfig } from './snmp/types.ts';
|
|
import { logger } from './logger.ts';
|
|
import { MigrationRunner } from './migrations/index.ts';
|
|
|
|
const execAsync = promisify(exec);
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
/**
|
|
* UPS configuration interface
|
|
*/
|
|
export interface IUpsConfig {
|
|
/** Unique ID for the UPS */
|
|
id: string;
|
|
/** Friendly name for the UPS */
|
|
name: string;
|
|
/** SNMP configuration settings */
|
|
snmp: ISnmpConfig;
|
|
/** Threshold settings for initiating shutdown */
|
|
thresholds: {
|
|
/** Shutdown when battery below this percentage */
|
|
battery: number;
|
|
/** Shutdown when runtime below this minutes */
|
|
runtime: number;
|
|
};
|
|
/** Group IDs this UPS belongs to */
|
|
groups: string[];
|
|
}
|
|
|
|
/**
|
|
* Group configuration interface
|
|
*/
|
|
export interface IGroupConfig {
|
|
/** Unique ID for the group */
|
|
id: string;
|
|
/** Friendly name for the group */
|
|
name: string;
|
|
/** Group operation mode */
|
|
mode: 'redundant' | 'nonRedundant';
|
|
/** Optional description */
|
|
description?: string;
|
|
}
|
|
|
|
/**
|
|
* Configuration interface for the daemon
|
|
*/
|
|
export interface INupstConfig {
|
|
/** Configuration format version */
|
|
version?: string;
|
|
/** UPS devices configuration */
|
|
upsDevices: IUpsConfig[];
|
|
/** Groups configuration */
|
|
groups: IGroupConfig[];
|
|
/** Check interval in milliseconds */
|
|
checkInterval: number;
|
|
|
|
// Legacy fields for backward compatibility (will be migrated away)
|
|
/** UPS list (v3 format - legacy) */
|
|
upsList?: IUpsConfig[];
|
|
/** SNMP configuration settings (v1 format - legacy) */
|
|
snmp?: ISnmpConfig;
|
|
/** Threshold settings (v1 format - legacy) */
|
|
thresholds?: {
|
|
/** Shutdown when battery below this percentage */
|
|
battery: number;
|
|
/** Shutdown when runtime below this minutes */
|
|
runtime: number;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* UPS status tracking interface
|
|
*/
|
|
interface IUpsStatus {
|
|
id: string;
|
|
name: string;
|
|
powerStatus: 'online' | 'onBattery' | 'unknown';
|
|
batteryCapacity: number;
|
|
batteryRuntime: number;
|
|
lastStatusChange: number;
|
|
lastCheckTime: number;
|
|
}
|
|
|
|
/**
|
|
* Daemon class for monitoring UPS and handling shutdown
|
|
* Responsible for loading/saving config and monitoring the UPS status
|
|
*/
|
|
export class NupstDaemon {
|
|
/** Default configuration path */
|
|
private readonly CONFIG_PATH = '/etc/nupst/config.json';
|
|
|
|
/** Default configuration */
|
|
private readonly DEFAULT_CONFIG: INupstConfig = {
|
|
version: '4.0',
|
|
upsDevices: [
|
|
{
|
|
id: 'default',
|
|
name: 'Default UPS',
|
|
snmp: {
|
|
host: '127.0.0.1',
|
|
port: 161,
|
|
community: 'public',
|
|
version: 1,
|
|
timeout: 5000,
|
|
// SNMPv3 defaults (used only if version === 3)
|
|
securityLevel: 'authPriv',
|
|
username: '',
|
|
authProtocol: 'SHA',
|
|
authKey: '',
|
|
privProtocol: 'AES',
|
|
privKey: '',
|
|
// UPS model for OID selection
|
|
upsModel: 'cyberpower',
|
|
},
|
|
thresholds: {
|
|
battery: 60, // Shutdown when battery below 60%
|
|
runtime: 20, // Shutdown when runtime below 20 minutes
|
|
},
|
|
groups: [],
|
|
},
|
|
],
|
|
groups: [],
|
|
checkInterval: 30000, // Check every 30 seconds
|
|
};
|
|
|
|
private config: INupstConfig;
|
|
private snmp: NupstSnmp;
|
|
private isRunning: boolean = false;
|
|
private upsStatus: Map<string, IUpsStatus> = new Map();
|
|
|
|
/**
|
|
* Create a new daemon instance with the given SNMP manager
|
|
*/
|
|
constructor(snmp: NupstSnmp) {
|
|
this.snmp = snmp;
|
|
this.config = this.DEFAULT_CONFIG;
|
|
}
|
|
|
|
/**
|
|
* Load configuration from file
|
|
* @throws Error if configuration file doesn't exist
|
|
*/
|
|
public async loadConfig(): Promise<INupstConfig> {
|
|
try {
|
|
// Check if config file exists
|
|
const configExists = fs.existsSync(this.CONFIG_PATH);
|
|
if (!configExists) {
|
|
const errorMsg = `No configuration found at ${this.CONFIG_PATH}`;
|
|
this.logConfigError(errorMsg);
|
|
throw new Error(errorMsg);
|
|
}
|
|
|
|
// Read and parse config
|
|
const configData = fs.readFileSync(this.CONFIG_PATH, 'utf8');
|
|
const parsedConfig = JSON.parse(configData);
|
|
|
|
// Run migrations to upgrade config format if needed
|
|
const migrationRunner = new MigrationRunner();
|
|
const { config: migratedConfig, migrated } = await migrationRunner.run(parsedConfig);
|
|
|
|
// Save migrated config back to disk if any migrations ran
|
|
if (migrated) {
|
|
this.config = migratedConfig;
|
|
await this.saveConfig(this.config);
|
|
} else {
|
|
this.config = migratedConfig;
|
|
}
|
|
|
|
return this.config;
|
|
} catch (error) {
|
|
if (
|
|
error instanceof Error && error.message && error.message.includes('No configuration found')
|
|
) {
|
|
throw error; // Re-throw the no configuration error
|
|
}
|
|
|
|
this.logConfigError(
|
|
`Error loading configuration: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
throw new Error('Failed to load configuration');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save configuration to file
|
|
*/
|
|
public saveConfig(config: INupstConfig): void {
|
|
try {
|
|
const configDir = path.dirname(this.CONFIG_PATH);
|
|
if (!fs.existsSync(configDir)) {
|
|
fs.mkdirSync(configDir, { recursive: true });
|
|
}
|
|
|
|
// Ensure version is always set and remove legacy fields before saving
|
|
const configToSave: INupstConfig = {
|
|
version: '4.0',
|
|
upsDevices: config.upsDevices,
|
|
groups: config.groups,
|
|
checkInterval: config.checkInterval,
|
|
};
|
|
|
|
fs.writeFileSync(this.CONFIG_PATH, JSON.stringify(configToSave, null, 2));
|
|
this.config = configToSave;
|
|
|
|
logger.logBox('Configuration Saved', [`Location: ${this.CONFIG_PATH}`], 45, 'success');
|
|
} catch (error) {
|
|
logger.error(`Error saving configuration: ${error}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper method to log configuration errors consistently
|
|
*/
|
|
private logConfigError(message: string): void {
|
|
logger.logBox('Configuration Error', [message, "Please run 'nupst setup' first to create a configuration."], 45, 'error');
|
|
}
|
|
|
|
/**
|
|
* Get the current configuration
|
|
*/
|
|
public getConfig(): INupstConfig {
|
|
return this.config;
|
|
}
|
|
|
|
/**
|
|
* Get the SNMP instance
|
|
*/
|
|
public getNupstSnmp(): NupstSnmp {
|
|
return this.snmp;
|
|
}
|
|
|
|
/**
|
|
* Start the monitoring daemon
|
|
*/
|
|
public async start(): Promise<void> {
|
|
if (this.isRunning) {
|
|
logger.log('Daemon is already running');
|
|
return;
|
|
}
|
|
|
|
logger.log('Starting NUPST daemon...');
|
|
|
|
try {
|
|
// Load configuration - this will throw an error if config doesn't exist
|
|
await this.loadConfig();
|
|
this.logConfigLoaded();
|
|
|
|
// Log version information
|
|
this.snmp.getNupst().logVersionInfo(false); // Don't check for updates immediately on startup
|
|
|
|
// Check for updates in the background
|
|
this.snmp.getNupst().checkForUpdates().then((updateAvailable: boolean) => {
|
|
if (updateAvailable) {
|
|
const updateStatus = this.snmp.getNupst().getUpdateStatus();
|
|
const boxWidth = 45;
|
|
logger.logBoxTitle('Update Available', boxWidth);
|
|
logger.logBoxLine(`Current Version: ${updateStatus.currentVersion}`);
|
|
logger.logBoxLine(`Latest Version: ${updateStatus.latestVersion}`);
|
|
logger.logBoxLine('Run "sudo nupst update" to update');
|
|
logger.logBoxEnd();
|
|
}
|
|
}).catch(() => {}); // Ignore errors checking for updates
|
|
|
|
// Initialize UPS status tracking
|
|
this.initializeUpsStatus();
|
|
|
|
// Start UPS monitoring
|
|
this.isRunning = true;
|
|
await this.monitor();
|
|
} catch (error) {
|
|
this.isRunning = false;
|
|
logger.error(
|
|
`Daemon failed to start: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
process.exit(1); // Exit with error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialize UPS status tracking for all UPS devices
|
|
*/
|
|
private initializeUpsStatus(): void {
|
|
this.upsStatus.clear();
|
|
|
|
if (this.config.upsDevices && this.config.upsDevices.length > 0) {
|
|
for (const ups of this.config.upsDevices) {
|
|
this.upsStatus.set(ups.id, {
|
|
id: ups.id,
|
|
name: ups.name,
|
|
powerStatus: 'unknown',
|
|
batteryCapacity: 100,
|
|
batteryRuntime: 999, // High value as default
|
|
lastStatusChange: Date.now(),
|
|
lastCheckTime: 0,
|
|
});
|
|
}
|
|
|
|
logger.log(`Initialized status tracking for ${this.config.upsDevices.length} UPS devices`);
|
|
} else {
|
|
logger.error('No UPS devices found in configuration');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Log the loaded configuration settings
|
|
*/
|
|
private logConfigLoaded(): void {
|
|
const boxWidth = 50;
|
|
logger.logBoxTitle('Configuration Loaded', boxWidth);
|
|
|
|
if (this.config.upsDevices && this.config.upsDevices.length > 0) {
|
|
logger.logBoxLine(`UPS Devices: ${this.config.upsDevices.length}`);
|
|
for (const ups of this.config.upsDevices) {
|
|
logger.logBoxLine(` - ${ups.name} (${ups.id}): ${ups.snmp.host}:${ups.snmp.port}`);
|
|
}
|
|
} else {
|
|
logger.logBoxLine('No UPS devices configured');
|
|
}
|
|
|
|
if (this.config.groups && this.config.groups.length > 0) {
|
|
logger.logBoxLine(`Groups: ${this.config.groups.length}`);
|
|
for (const group of this.config.groups) {
|
|
logger.logBoxLine(` - ${group.name} (${group.id}): ${group.mode} mode`);
|
|
}
|
|
} else {
|
|
logger.logBoxLine('No Groups configured');
|
|
}
|
|
|
|
logger.logBoxLine(`Check Interval: ${this.config.checkInterval / 1000} seconds`);
|
|
logger.logBoxEnd();
|
|
}
|
|
|
|
/**
|
|
* Stop the monitoring daemon
|
|
*/
|
|
public stop(): void {
|
|
logger.log('Stopping NUPST daemon...');
|
|
this.isRunning = false;
|
|
}
|
|
|
|
/**
|
|
* Monitor the UPS status and trigger shutdown when necessary
|
|
*/
|
|
private async monitor(): Promise<void> {
|
|
logger.log('Starting UPS monitoring...');
|
|
|
|
if (!this.config.upsDevices || this.config.upsDevices.length === 0) {
|
|
logger.warn('No UPS devices found in configuration. Daemon will remain idle...');
|
|
// Don't exit - enter idle monitoring mode instead
|
|
await this.idleMonitoring();
|
|
return;
|
|
}
|
|
|
|
let lastLogTime = 0; // Track when we last logged status
|
|
const LOG_INTERVAL = 5 * 60 * 1000; // Log at least every 5 minutes (300000ms)
|
|
|
|
// Monitor continuously
|
|
while (this.isRunning) {
|
|
try {
|
|
// Check all UPS devices
|
|
await this.checkAllUpsDevices();
|
|
|
|
// Log periodic status update
|
|
const currentTime = Date.now();
|
|
if (currentTime - lastLogTime >= LOG_INTERVAL) {
|
|
this.logAllUpsStatus();
|
|
lastLogTime = currentTime;
|
|
}
|
|
|
|
// Check if shutdown is required based on group configurations
|
|
await this.evaluateGroupShutdownConditions();
|
|
|
|
// Wait before next check
|
|
await this.sleep(this.config.checkInterval);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error during UPS monitoring: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
await this.sleep(this.config.checkInterval);
|
|
}
|
|
}
|
|
|
|
logger.log('UPS monitoring stopped');
|
|
}
|
|
|
|
/**
|
|
* Check status of all UPS devices
|
|
*/
|
|
private async checkAllUpsDevices(): Promise<void> {
|
|
for (const ups of this.config.upsDevices) {
|
|
try {
|
|
const upsStatus = this.upsStatus.get(ups.id);
|
|
if (!upsStatus) {
|
|
// Initialize status for this UPS if not exists
|
|
this.upsStatus.set(ups.id, {
|
|
id: ups.id,
|
|
name: ups.name,
|
|
powerStatus: 'unknown',
|
|
batteryCapacity: 100,
|
|
batteryRuntime: 999,
|
|
lastStatusChange: Date.now(),
|
|
lastCheckTime: 0,
|
|
});
|
|
}
|
|
|
|
// Check UPS status
|
|
const status = await this.snmp.getUpsStatus(ups.snmp);
|
|
const currentTime = Date.now();
|
|
|
|
// Get the current status from the map
|
|
const currentStatus = this.upsStatus.get(ups.id);
|
|
|
|
// Update status with new values
|
|
const updatedStatus: IUpsStatus = {
|
|
id: ups.id,
|
|
name: ups.name,
|
|
powerStatus: status.powerStatus,
|
|
batteryCapacity: status.batteryCapacity,
|
|
batteryRuntime: status.batteryRuntime,
|
|
lastCheckTime: currentTime,
|
|
lastStatusChange: currentStatus?.lastStatusChange || currentTime,
|
|
};
|
|
|
|
// Check if power status changed
|
|
if (currentStatus && currentStatus.powerStatus !== status.powerStatus) {
|
|
logger.logBoxTitle(`Power Status Change: ${ups.name}`, 50);
|
|
logger.logBoxLine(`Status changed: ${currentStatus.powerStatus} → ${status.powerStatus}`);
|
|
logger.logBoxEnd();
|
|
|
|
updatedStatus.lastStatusChange = currentTime;
|
|
}
|
|
|
|
// Update the status in the map
|
|
this.upsStatus.set(ups.id, updatedStatus);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error checking UPS ${ups.name} (${ups.id}): ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Log status of all UPS devices
|
|
*/
|
|
private logAllUpsStatus(): void {
|
|
const timestamp = new Date().toISOString();
|
|
const boxWidth = 60;
|
|
logger.logBoxTitle('Periodic Status Update', boxWidth);
|
|
logger.logBoxLine(`Timestamp: ${timestamp}`);
|
|
logger.logBoxLine('');
|
|
|
|
for (const [id, status] of this.upsStatus.entries()) {
|
|
logger.logBoxLine(`UPS: ${status.name} (${id})`);
|
|
logger.logBoxLine(` Power Status: ${status.powerStatus}`);
|
|
logger.logBoxLine(
|
|
` Battery: ${status.batteryCapacity}% | Runtime: ${status.batteryRuntime} min`,
|
|
);
|
|
logger.logBoxLine('');
|
|
}
|
|
|
|
logger.logBoxEnd();
|
|
}
|
|
|
|
/**
|
|
* Evaluate if shutdown is required based on group configurations
|
|
*/
|
|
private async evaluateGroupShutdownConditions(): Promise<void> {
|
|
if (!this.config.groups || this.config.groups.length === 0) {
|
|
// No groups defined, check individual UPS conditions
|
|
for (const [id, status] of this.upsStatus.entries()) {
|
|
if (status.powerStatus === 'onBattery') {
|
|
// Find the UPS config
|
|
const ups = this.config.upsDevices.find((u) => u.id === id);
|
|
if (ups) {
|
|
await this.evaluateUpsShutdownCondition(ups, status);
|
|
}
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Evaluate each group
|
|
for (const group of this.config.groups) {
|
|
// Find all UPS devices in this group
|
|
const upsDevicesInGroup = this.config.upsDevices.filter((ups) =>
|
|
ups.groups && ups.groups.includes(group.id)
|
|
);
|
|
|
|
if (upsDevicesInGroup.length === 0) {
|
|
// No UPS devices in this group
|
|
continue;
|
|
}
|
|
|
|
if (group.mode === 'redundant') {
|
|
// Redundant mode: only shutdown if ALL UPS devices in the group are in critical condition
|
|
await this.evaluateRedundantGroup(group, upsDevicesInGroup);
|
|
} else {
|
|
// Non-redundant mode: shutdown if ANY UPS device in the group is in critical condition
|
|
await this.evaluateNonRedundantGroup(group, upsDevicesInGroup);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Evaluate a redundant group for shutdown conditions
|
|
* In redundant mode, we only shut down if ALL UPS devices are in critical condition
|
|
*/
|
|
private async evaluateRedundantGroup(
|
|
group: IGroupConfig,
|
|
upsDevices: IUpsConfig[],
|
|
): Promise<void> {
|
|
// Count UPS devices on battery and in critical condition
|
|
let upsOnBattery = 0;
|
|
let upsInCriticalCondition = 0;
|
|
|
|
for (const ups of upsDevices) {
|
|
const status = this.upsStatus.get(ups.id);
|
|
if (!status) continue;
|
|
|
|
if (status.powerStatus === 'onBattery') {
|
|
upsOnBattery++;
|
|
|
|
// Check if this UPS is in critical condition
|
|
if (
|
|
status.batteryCapacity < ups.thresholds.battery ||
|
|
status.batteryRuntime < ups.thresholds.runtime
|
|
) {
|
|
upsInCriticalCondition++;
|
|
}
|
|
}
|
|
}
|
|
|
|
// All UPS devices must be online for a redundant group to be considered healthy
|
|
const allUpsCount = upsDevices.length;
|
|
|
|
// If all UPS are on battery and in critical condition, shutdown
|
|
if (upsOnBattery === allUpsCount && upsInCriticalCondition === allUpsCount) {
|
|
logger.logBoxTitle(`Group Shutdown Required: ${group.name}`, 50);
|
|
logger.logBoxLine(`Mode: Redundant`);
|
|
logger.logBoxLine(`All ${allUpsCount} UPS devices in critical condition`);
|
|
logger.logBoxEnd();
|
|
|
|
await this.initiateShutdown(
|
|
`All UPS devices in redundant group "${group.name}" in critical condition`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Evaluate a non-redundant group for shutdown conditions
|
|
* In non-redundant mode, we shut down if ANY UPS device is in critical condition
|
|
*/
|
|
private async evaluateNonRedundantGroup(
|
|
group: IGroupConfig,
|
|
upsDevices: IUpsConfig[],
|
|
): Promise<void> {
|
|
for (const ups of upsDevices) {
|
|
const status = this.upsStatus.get(ups.id);
|
|
if (!status) continue;
|
|
|
|
if (status.powerStatus === 'onBattery') {
|
|
// Check if this UPS is in critical condition
|
|
if (
|
|
status.batteryCapacity < ups.thresholds.battery ||
|
|
status.batteryRuntime < ups.thresholds.runtime
|
|
) {
|
|
logger.logBoxTitle(`Group Shutdown Required: ${group.name}`, 50);
|
|
logger.logBoxLine(`Mode: Non-Redundant`);
|
|
logger.logBoxLine(`UPS ${ups.name} in critical condition`);
|
|
logger.logBoxLine(
|
|
`Battery: ${status.batteryCapacity}% (threshold: ${ups.thresholds.battery}%)`,
|
|
);
|
|
logger.logBoxLine(
|
|
`Runtime: ${status.batteryRuntime} min (threshold: ${ups.thresholds.runtime} min)`,
|
|
);
|
|
logger.logBoxEnd();
|
|
|
|
await this.initiateShutdown(
|
|
`UPS "${ups.name}" in non-redundant group "${group.name}" in critical condition`,
|
|
);
|
|
return; // Exit after initiating shutdown
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Evaluate an individual UPS for shutdown conditions
|
|
*/
|
|
private async evaluateUpsShutdownCondition(ups: IUpsConfig, status: IUpsStatus): Promise<void> {
|
|
// Only evaluate UPS devices not in any group
|
|
if (ups.groups && ups.groups.length > 0) {
|
|
return;
|
|
}
|
|
|
|
// Check threshold conditions
|
|
if (
|
|
status.batteryCapacity < ups.thresholds.battery ||
|
|
status.batteryRuntime < ups.thresholds.runtime
|
|
) {
|
|
logger.logBoxTitle(`UPS Shutdown Required: ${ups.name}`, 50);
|
|
logger.logBoxLine(
|
|
`Battery: ${status.batteryCapacity}% (threshold: ${ups.thresholds.battery}%)`,
|
|
);
|
|
logger.logBoxLine(
|
|
`Runtime: ${status.batteryRuntime} min (threshold: ${ups.thresholds.runtime} min)`,
|
|
);
|
|
logger.logBoxEnd();
|
|
|
|
await this.initiateShutdown(`UPS "${ups.name}" battery or runtime below threshold`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initiate system shutdown with UPS monitoring during shutdown
|
|
* @param reason Reason for shutdown
|
|
*/
|
|
public async initiateShutdown(reason: string): Promise<void> {
|
|
logger.log(`Initiating system shutdown due to: ${reason}`);
|
|
|
|
// Set a longer delay for shutdown to allow VMs and services to close
|
|
const shutdownDelayMinutes = 5;
|
|
|
|
try {
|
|
// Find shutdown command in common system paths
|
|
const shutdownPaths = [
|
|
'/sbin/shutdown',
|
|
'/usr/sbin/shutdown',
|
|
'/bin/shutdown',
|
|
'/usr/bin/shutdown',
|
|
];
|
|
|
|
let shutdownCmd = '';
|
|
for (const path of shutdownPaths) {
|
|
try {
|
|
if (fs.existsSync(path)) {
|
|
shutdownCmd = path;
|
|
logger.log(`Found shutdown command at: ${shutdownCmd}`);
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
// Continue checking other paths
|
|
}
|
|
}
|
|
|
|
if (shutdownCmd) {
|
|
// Execute shutdown command with delay to allow for VM graceful shutdown
|
|
logger.log(
|
|
`Executing: ${shutdownCmd} -h +${shutdownDelayMinutes} "UPS battery critical..."`,
|
|
);
|
|
const { stdout } = await execFileAsync(shutdownCmd, [
|
|
'-h',
|
|
`+${shutdownDelayMinutes}`,
|
|
`UPS battery critical, shutting down in ${shutdownDelayMinutes} minutes`,
|
|
]);
|
|
logger.log(`Shutdown initiated: ${stdout}`);
|
|
logger.log(`Allowing ${shutdownDelayMinutes} minutes for VMs to shut down safely`);
|
|
} else {
|
|
// Try using the PATH to find shutdown
|
|
try {
|
|
logger.log('Shutdown command not found in common paths, trying via PATH...');
|
|
const { stdout } = await execAsync(
|
|
`shutdown -h +${shutdownDelayMinutes} "UPS battery critical, shutting down in ${shutdownDelayMinutes} minutes"`,
|
|
{
|
|
env: process.env, // Pass the current environment
|
|
},
|
|
);
|
|
logger.log(`Shutdown initiated: ${stdout}`);
|
|
} catch (e) {
|
|
throw new Error(
|
|
`Shutdown command not found: ${e instanceof Error ? e.message : String(e)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// Monitor UPS during shutdown and force immediate shutdown if battery gets too low
|
|
logger.log('Monitoring UPS during shutdown process...');
|
|
await this.monitorDuringShutdown();
|
|
} catch (error) {
|
|
logger.error(`Failed to initiate shutdown: ${error}`);
|
|
|
|
// Try alternative shutdown methods
|
|
const alternatives = [
|
|
{ cmd: 'poweroff', args: ['--force'] },
|
|
{ cmd: 'halt', args: ['-p'] },
|
|
{ cmd: 'systemctl', args: ['poweroff'] },
|
|
{ cmd: 'reboot', args: ['-p'] }, // Some systems allow reboot -p for power off
|
|
];
|
|
|
|
for (const alt of alternatives) {
|
|
try {
|
|
// First check if command exists in common system paths
|
|
const paths = [
|
|
`/sbin/${alt.cmd}`,
|
|
`/usr/sbin/${alt.cmd}`,
|
|
`/bin/${alt.cmd}`,
|
|
`/usr/bin/${alt.cmd}`,
|
|
];
|
|
|
|
let cmdPath = '';
|
|
for (const path of paths) {
|
|
if (fs.existsSync(path)) {
|
|
cmdPath = path;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (cmdPath) {
|
|
logger.log(`Trying alternative shutdown method: ${cmdPath} ${alt.args.join(' ')}`);
|
|
await execFileAsync(cmdPath, alt.args);
|
|
return; // Exit if successful
|
|
} else {
|
|
// Try using PATH environment
|
|
logger.log(`Trying alternative via PATH: ${alt.cmd} ${alt.args.join(' ')}`);
|
|
await execAsync(`${alt.cmd} ${alt.args.join(' ')}`, {
|
|
env: process.env, // Pass the current environment
|
|
});
|
|
return; // Exit if successful
|
|
}
|
|
} catch (altError) {
|
|
logger.error(`Alternative method ${alt.cmd} failed: ${altError}`);
|
|
// Continue to next method
|
|
}
|
|
}
|
|
|
|
logger.error('All shutdown methods failed');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Monitor UPS during system shutdown
|
|
* Force immediate shutdown if any UPS gets critically low
|
|
*/
|
|
private async monitorDuringShutdown(): Promise<void> {
|
|
const EMERGENCY_RUNTIME_THRESHOLD = 5; // 5 minutes remaining is critical
|
|
const CHECK_INTERVAL = 30000; // Check every 30 seconds during shutdown
|
|
const MAX_MONITORING_TIME = 5 * 60 * 1000; // Max 5 minutes of monitoring
|
|
const startTime = Date.now();
|
|
|
|
logger.log(
|
|
`Emergency shutdown threshold: ${EMERGENCY_RUNTIME_THRESHOLD} minutes remaining battery runtime`,
|
|
);
|
|
|
|
// Continue monitoring until max monitoring time is reached
|
|
while (Date.now() - startTime < MAX_MONITORING_TIME) {
|
|
try {
|
|
logger.log('Checking UPS status during shutdown...');
|
|
|
|
// Check all UPS devices
|
|
for (const ups of this.config.upsDevices) {
|
|
try {
|
|
const status = await this.snmp.getUpsStatus(ups.snmp);
|
|
|
|
logger.log(
|
|
`UPS ${ups.name}: Battery ${status.batteryCapacity}%, Runtime: ${status.batteryRuntime} minutes`,
|
|
);
|
|
|
|
// If any UPS battery runtime gets critically low, force immediate shutdown
|
|
if (status.batteryRuntime < EMERGENCY_RUNTIME_THRESHOLD) {
|
|
logger.logBoxTitle('EMERGENCY SHUTDOWN', 50);
|
|
logger.logBoxLine(
|
|
`UPS ${ups.name} runtime critically low: ${status.batteryRuntime} minutes`,
|
|
);
|
|
logger.logBoxLine('Forcing immediate shutdown!');
|
|
logger.logBoxEnd();
|
|
|
|
// Force immediate shutdown
|
|
await this.forceImmediateShutdown();
|
|
return;
|
|
}
|
|
} catch (upsError) {
|
|
logger.error(
|
|
`Error checking UPS ${ups.name} during shutdown: ${
|
|
upsError instanceof Error ? upsError.message : String(upsError)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// Wait before checking again
|
|
await this.sleep(CHECK_INTERVAL);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error monitoring UPS during shutdown: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
await this.sleep(CHECK_INTERVAL);
|
|
}
|
|
}
|
|
|
|
logger.log('UPS monitoring during shutdown completed');
|
|
}
|
|
|
|
/**
|
|
* Force an immediate system shutdown
|
|
*/
|
|
private async forceImmediateShutdown(): Promise<void> {
|
|
try {
|
|
// Find shutdown command in common system paths
|
|
const shutdownPaths = [
|
|
'/sbin/shutdown',
|
|
'/usr/sbin/shutdown',
|
|
'/bin/shutdown',
|
|
'/usr/bin/shutdown',
|
|
];
|
|
|
|
let shutdownCmd = '';
|
|
for (const path of shutdownPaths) {
|
|
if (fs.existsSync(path)) {
|
|
shutdownCmd = path;
|
|
logger.log(`Found shutdown command at: ${shutdownCmd}`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (shutdownCmd) {
|
|
logger.log(`Executing emergency shutdown: ${shutdownCmd} -h now`);
|
|
await execFileAsync(shutdownCmd, [
|
|
'-h',
|
|
'now',
|
|
'EMERGENCY: UPS battery critically low, shutting down NOW',
|
|
]);
|
|
} else {
|
|
// Try using the PATH to find shutdown
|
|
logger.log('Shutdown command not found in common paths, trying via PATH...');
|
|
await execAsync(
|
|
'shutdown -h now "EMERGENCY: UPS battery critically low, shutting down NOW"',
|
|
{
|
|
env: process.env, // Pass the current environment
|
|
},
|
|
);
|
|
}
|
|
} catch (error) {
|
|
logger.error('Emergency shutdown failed, trying alternative methods...');
|
|
|
|
// Try alternative shutdown methods in sequence
|
|
const alternatives = [
|
|
{ cmd: 'poweroff', args: ['--force'] },
|
|
{ cmd: 'halt', args: ['-p'] },
|
|
{ cmd: 'systemctl', args: ['poweroff'] },
|
|
];
|
|
|
|
for (const alt of alternatives) {
|
|
try {
|
|
// Check common paths
|
|
const paths = [
|
|
`/sbin/${alt.cmd}`,
|
|
`/usr/sbin/${alt.cmd}`,
|
|
`/bin/${alt.cmd}`,
|
|
`/usr/bin/${alt.cmd}`,
|
|
];
|
|
|
|
let cmdPath = '';
|
|
for (const path of paths) {
|
|
if (fs.existsSync(path)) {
|
|
cmdPath = path;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (cmdPath) {
|
|
logger.log(`Emergency: using ${cmdPath} ${alt.args.join(' ')}`);
|
|
await execFileAsync(cmdPath, alt.args);
|
|
return; // Exit if successful
|
|
} else {
|
|
// Try using PATH
|
|
logger.log(`Emergency: trying ${alt.cmd} via PATH`);
|
|
await execAsync(`${alt.cmd} ${alt.args.join(' ')}`, {
|
|
env: process.env,
|
|
});
|
|
return; // Exit if successful
|
|
}
|
|
} catch (altError) {
|
|
// Continue to next method
|
|
}
|
|
}
|
|
|
|
logger.error('All emergency shutdown methods failed');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Idle monitoring loop when no UPS devices are configured
|
|
* Watches for config changes and reloads when detected
|
|
*/
|
|
private async idleMonitoring(): Promise<void> {
|
|
const IDLE_CHECK_INTERVAL = 60000; // Check every 60 seconds
|
|
let lastConfigCheck = Date.now();
|
|
const CONFIG_CHECK_INTERVAL = 60000; // Check config every minute
|
|
|
|
logger.log('Entering idle monitoring mode...');
|
|
logger.log('Daemon will check for config changes every 60 seconds');
|
|
|
|
// Start file watcher for hot-reload
|
|
this.watchConfigFile();
|
|
|
|
while (this.isRunning) {
|
|
try {
|
|
const currentTime = Date.now();
|
|
|
|
// Periodically check if config has been updated
|
|
if (currentTime - lastConfigCheck >= CONFIG_CHECK_INTERVAL) {
|
|
try {
|
|
// Try to load config
|
|
const newConfig = await this.loadConfig();
|
|
|
|
// Check if we now have UPS devices configured
|
|
if (newConfig.upsDevices && newConfig.upsDevices.length > 0) {
|
|
logger.success('Configuration updated! UPS devices found. Starting monitoring...');
|
|
this.initializeUpsStatus();
|
|
// Exit idle mode and start monitoring
|
|
await this.monitor();
|
|
return;
|
|
}
|
|
} catch (error) {
|
|
// Config still doesn't exist or invalid, continue waiting
|
|
}
|
|
|
|
lastConfigCheck = currentTime;
|
|
}
|
|
|
|
await this.sleep(IDLE_CHECK_INTERVAL);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error during idle monitoring: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
await this.sleep(IDLE_CHECK_INTERVAL);
|
|
}
|
|
}
|
|
|
|
logger.log('Idle monitoring stopped');
|
|
}
|
|
|
|
/**
|
|
* Watch config file for changes and reload automatically
|
|
*/
|
|
private watchConfigFile(): void {
|
|
try {
|
|
// Use Deno's file watcher to monitor config file
|
|
const configDir = path.dirname(this.CONFIG_PATH);
|
|
|
|
// Spawn a background watcher (non-blocking)
|
|
(async () => {
|
|
try {
|
|
const watcher = Deno.watchFs(configDir);
|
|
|
|
logger.log('Config file watcher started');
|
|
|
|
for await (const event of watcher) {
|
|
// Only respond to modify events on the config file
|
|
if (
|
|
event.kind === 'modify' &&
|
|
event.paths.some((p) => p.includes('config.json'))
|
|
) {
|
|
logger.info('Config file changed, reloading...');
|
|
await this.reloadConfig();
|
|
}
|
|
|
|
// Stop watching if daemon stopped
|
|
if (!this.isRunning) {
|
|
break;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
// Watcher error - not critical, just log it
|
|
logger.dim(
|
|
`Config watcher stopped: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
})();
|
|
} catch (error) {
|
|
// If we can't start the watcher, just log and continue
|
|
// The periodic check will still work
|
|
logger.dim('Could not start config file watcher, using periodic checks only');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reload configuration and restart monitoring if needed
|
|
*/
|
|
private async reloadConfig(): Promise<void> {
|
|
try {
|
|
const oldDeviceCount = this.config.upsDevices?.length || 0;
|
|
|
|
// Load the new configuration
|
|
await this.loadConfig();
|
|
const newDeviceCount = this.config.upsDevices?.length || 0;
|
|
|
|
if (newDeviceCount > 0 && oldDeviceCount === 0) {
|
|
logger.success(`Configuration reloaded! Found ${newDeviceCount} UPS device(s)`);
|
|
logger.info('Monitoring will start automatically...');
|
|
} else if (newDeviceCount !== oldDeviceCount) {
|
|
logger.success(
|
|
`Configuration reloaded! UPS devices: ${oldDeviceCount} → ${newDeviceCount}`,
|
|
);
|
|
|
|
// Reinitialize UPS status tracking
|
|
this.initializeUpsStatus();
|
|
} else {
|
|
logger.success('Configuration reloaded successfully');
|
|
}
|
|
} catch (error) {
|
|
logger.warn(
|
|
`Failed to reload config: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sleep for the specified milliseconds
|
|
*/
|
|
private sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
}
|