1084 lines
35 KiB
TypeScript
1084 lines
35 KiB
TypeScript
import process from 'node:process';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { NupstSnmp } from './snmp/manager.ts';
|
|
import type { ISnmpConfig, IUpsStatus as ISnmpUpsStatus } from './snmp/types.ts';
|
|
import { NupstUpsd } from './upsd/client.ts';
|
|
import type { IUpsdConfig } from './upsd/types.ts';
|
|
import type { TProtocol } from './protocol/types.ts';
|
|
import { ProtocolResolver } from './protocol/resolver.ts';
|
|
import { logger } from './logger.ts';
|
|
import { MigrationRunner } from './migrations/index.ts';
|
|
import { formatPowerStatus, getBatteryColor, getRuntimeColor, theme } from './colors.ts';
|
|
import type { IActionConfig } from './actions/base-action.ts';
|
|
import { ActionManager } from './actions/index.ts';
|
|
import {
|
|
applyDefaultShutdownDelay,
|
|
decideUpsActionExecution,
|
|
type TUpsTriggerReason,
|
|
} from './action-orchestration.ts';
|
|
import { NupstHttpServer } from './http-server.ts';
|
|
import { NETWORK, PAUSE, SHUTDOWN, THRESHOLDS, TIMING, UI } from './constants.ts';
|
|
import {
|
|
analyzeConfigReload,
|
|
shouldRefreshPauseState,
|
|
shouldReloadConfig,
|
|
} from './config-watch.ts';
|
|
import { type IPauseState, loadPauseSnapshot } from './pause-state.ts';
|
|
import { ShutdownExecutor } from './shutdown-executor.ts';
|
|
import {
|
|
buildFailedUpsPollSnapshot,
|
|
buildSuccessfulUpsPollSnapshot,
|
|
ensureUpsStatus,
|
|
hasThresholdViolation,
|
|
} from './ups-monitoring.ts';
|
|
import {
|
|
buildShutdownErrorRow,
|
|
buildShutdownStatusRow,
|
|
selectEmergencyCandidate,
|
|
} from './shutdown-monitoring.ts';
|
|
import { createInitialUpsStatus, type IUpsStatus } from './ups-status.ts';
|
|
|
|
/**
|
|
* UPS configuration interface
|
|
*/
|
|
export interface IUpsConfig {
|
|
/** Unique ID for the UPS */
|
|
id: string;
|
|
/** Friendly name for the UPS */
|
|
name: string;
|
|
/** Communication protocol (defaults to 'snmp') */
|
|
protocol?: TProtocol;
|
|
/** SNMP configuration settings (required for 'snmp' protocol) */
|
|
snmp?: ISnmpConfig;
|
|
/** UPSD/NIS configuration settings (required for 'upsd' protocol) */
|
|
upsd?: IUpsdConfig;
|
|
/** Group IDs this UPS belongs to */
|
|
groups: string[];
|
|
/** Actions to trigger on power status changes and threshold violations */
|
|
actions?: IActionConfig[];
|
|
}
|
|
|
|
/**
|
|
* Group configuration interface
|
|
*/
|
|
export interface IGroupConfig {
|
|
/** Unique ID for the group */
|
|
id: string;
|
|
/** Friendly name for the group */
|
|
name: string;
|
|
/** Group operation mode */
|
|
mode: 'redundant' | 'nonRedundant';
|
|
/** Optional description */
|
|
description?: string;
|
|
/** Actions to trigger on power status changes and threshold violations */
|
|
actions?: IActionConfig[];
|
|
}
|
|
|
|
/**
|
|
* HTTP Server configuration interface
|
|
*/
|
|
export interface IHttpServerConfig {
|
|
/** Whether HTTP server is enabled */
|
|
enabled: boolean;
|
|
/** Port to listen on */
|
|
port: number;
|
|
/** URL path for the endpoint */
|
|
path: string;
|
|
/** Authentication token */
|
|
authToken: string;
|
|
}
|
|
|
|
/**
|
|
* Configuration interface for the daemon
|
|
*/
|
|
export interface INupstConfig {
|
|
/** Configuration format version */
|
|
version?: string;
|
|
/** UPS devices configuration */
|
|
upsDevices: IUpsConfig[];
|
|
/** Groups configuration */
|
|
groups: IGroupConfig[];
|
|
/** Check interval in milliseconds */
|
|
checkInterval: number;
|
|
/** Default delay in minutes for shutdown actions without an override */
|
|
defaultShutdownDelay?: number;
|
|
/** HTTP Server configuration */
|
|
httpServer?: IHttpServerConfig;
|
|
|
|
// Legacy fields for backward compatibility (will be migrated away)
|
|
/** UPS list (v3 format - legacy) */
|
|
upsList?: IUpsConfig[];
|
|
/** SNMP configuration settings (v1 format - legacy) */
|
|
snmp?: ISnmpConfig;
|
|
/** Threshold settings (v1 format - legacy) */
|
|
thresholds?: {
|
|
/** Shutdown when battery below this percentage */
|
|
battery: number;
|
|
/** Shutdown when runtime below this minutes */
|
|
runtime: number;
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Daemon class for monitoring UPS and handling shutdown
|
|
* Responsible for loading/saving config and monitoring the UPS status
|
|
*/
|
|
export class NupstDaemon {
|
|
/** Default configuration path */
|
|
private readonly CONFIG_PATH = '/etc/nupst/config.json';
|
|
|
|
/** Default configuration */
|
|
private readonly DEFAULT_CONFIG: INupstConfig = {
|
|
version: '4.3',
|
|
defaultShutdownDelay: SHUTDOWN.DEFAULT_DELAY_MINUTES,
|
|
upsDevices: [
|
|
{
|
|
id: 'default',
|
|
name: 'Default UPS',
|
|
snmp: {
|
|
host: '127.0.0.1',
|
|
port: 161,
|
|
community: 'public',
|
|
version: 1,
|
|
timeout: 5000,
|
|
// SNMPv3 defaults (used only if version === 3)
|
|
securityLevel: 'authPriv',
|
|
username: '',
|
|
authProtocol: 'SHA',
|
|
authKey: '',
|
|
privProtocol: 'AES',
|
|
privKey: '',
|
|
// UPS model for OID selection
|
|
upsModel: 'cyberpower',
|
|
runtimeUnit: 'ticks',
|
|
},
|
|
groups: [],
|
|
actions: [
|
|
{
|
|
type: 'shutdown',
|
|
triggerMode: 'onlyThresholds',
|
|
thresholds: {
|
|
battery: THRESHOLDS.DEFAULT_BATTERY_PERCENT, // Shutdown when battery below 60%
|
|
runtime: THRESHOLDS.DEFAULT_RUNTIME_MINUTES, // Shutdown when runtime below 20 minutes
|
|
},
|
|
},
|
|
],
|
|
},
|
|
],
|
|
groups: [],
|
|
checkInterval: TIMING.CHECK_INTERVAL_MS, // Check every 30 seconds
|
|
};
|
|
|
|
private config: INupstConfig;
|
|
private snmp: NupstSnmp;
|
|
private upsd: NupstUpsd;
|
|
private protocolResolver: ProtocolResolver;
|
|
private isRunning: boolean = false;
|
|
private isPaused: boolean = false;
|
|
private pauseState: IPauseState | null = null;
|
|
private upsStatus: Map<string, IUpsStatus> = new Map();
|
|
private httpServer?: NupstHttpServer;
|
|
private readonly shutdownExecutor: ShutdownExecutor;
|
|
|
|
/**
|
|
* Create a new daemon instance with the given protocol managers
|
|
*/
|
|
constructor(snmp: NupstSnmp, upsd: NupstUpsd) {
|
|
this.snmp = snmp;
|
|
this.upsd = upsd;
|
|
this.protocolResolver = new ProtocolResolver(snmp, upsd);
|
|
this.shutdownExecutor = new ShutdownExecutor();
|
|
this.config = this.DEFAULT_CONFIG;
|
|
}
|
|
|
|
/**
|
|
* Load configuration from file
|
|
* @throws Error if configuration file doesn't exist
|
|
*/
|
|
public async loadConfig(): Promise<INupstConfig> {
|
|
try {
|
|
// Check if config file exists
|
|
const configExists = fs.existsSync(this.CONFIG_PATH);
|
|
if (!configExists) {
|
|
const errorMsg = `No configuration found at ${this.CONFIG_PATH}`;
|
|
this.logConfigError(errorMsg);
|
|
throw new Error(errorMsg);
|
|
}
|
|
|
|
// Read and parse config
|
|
const configData = fs.readFileSync(this.CONFIG_PATH, 'utf8');
|
|
const parsedConfig = JSON.parse(configData);
|
|
|
|
// Run migrations to upgrade config format if needed
|
|
const migrationRunner = new MigrationRunner();
|
|
const { config: migratedConfig, migrated } = await migrationRunner.run(parsedConfig);
|
|
|
|
// Save migrated or normalized config back to disk when needed.
|
|
// Cast to INupstConfig since migrations ensure the output is valid.
|
|
const validConfig = migratedConfig as unknown as INupstConfig;
|
|
const normalizedShutdownDelay = this.normalizeShutdownDelay(validConfig.defaultShutdownDelay);
|
|
const shouldPersistNormalizedConfig = validConfig.defaultShutdownDelay !== normalizedShutdownDelay;
|
|
validConfig.defaultShutdownDelay = normalizedShutdownDelay;
|
|
if (migrated || shouldPersistNormalizedConfig) {
|
|
this.config = validConfig;
|
|
await this.saveConfig(this.config);
|
|
} else {
|
|
this.config = validConfig;
|
|
}
|
|
|
|
return this.config;
|
|
} catch (error) {
|
|
if (
|
|
error instanceof Error && error.message && error.message.includes('No configuration found')
|
|
) {
|
|
throw error; // Re-throw the no configuration error
|
|
}
|
|
|
|
this.logConfigError(
|
|
`Error loading configuration: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
throw new Error('Failed to load configuration');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save configuration to file
|
|
*/
|
|
public saveConfig(config: INupstConfig): void {
|
|
try {
|
|
const configDir = path.dirname(this.CONFIG_PATH);
|
|
if (!fs.existsSync(configDir)) {
|
|
fs.mkdirSync(configDir, { recursive: true });
|
|
}
|
|
|
|
// Ensure version is always set and remove legacy fields before saving
|
|
const configToSave: INupstConfig = {
|
|
version: '4.3',
|
|
upsDevices: config.upsDevices,
|
|
groups: config.groups,
|
|
checkInterval: config.checkInterval,
|
|
defaultShutdownDelay: this.normalizeShutdownDelay(config.defaultShutdownDelay),
|
|
...(config.httpServer ? { httpServer: config.httpServer } : {}),
|
|
};
|
|
|
|
fs.writeFileSync(this.CONFIG_PATH, JSON.stringify(configToSave, null, 2));
|
|
this.config = configToSave;
|
|
|
|
logger.logBox('Configuration Saved', [`Location: ${this.CONFIG_PATH}`], 45, 'success');
|
|
} catch (error) {
|
|
logger.error(`Error saving configuration: ${error}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Helper method to log configuration errors consistently
|
|
*/
|
|
private logConfigError(message: string): void {
|
|
logger.logBox(
|
|
'Configuration Error',
|
|
[message, "Please run 'nupst ups add' first to create a configuration."],
|
|
45,
|
|
'error',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get the current configuration
|
|
*/
|
|
public getConfig(): INupstConfig {
|
|
return this.config;
|
|
}
|
|
|
|
private normalizeShutdownDelay(delayMinutes: number | undefined): number {
|
|
if (
|
|
typeof delayMinutes !== 'number' ||
|
|
!Number.isFinite(delayMinutes) ||
|
|
delayMinutes < 0
|
|
) {
|
|
return SHUTDOWN.DEFAULT_DELAY_MINUTES;
|
|
}
|
|
|
|
return delayMinutes;
|
|
}
|
|
|
|
private getDefaultShutdownDelayMinutes(): number {
|
|
return this.normalizeShutdownDelay(this.config.defaultShutdownDelay);
|
|
}
|
|
|
|
/**
|
|
* Get the SNMP instance
|
|
*/
|
|
public getNupstSnmp(): NupstSnmp {
|
|
return this.snmp;
|
|
}
|
|
|
|
/**
|
|
* Get the UPSD instance
|
|
*/
|
|
public getNupstUpsd(): NupstUpsd {
|
|
return this.upsd;
|
|
}
|
|
|
|
/**
|
|
* Start the monitoring daemon
|
|
*/
|
|
public async start(): Promise<void> {
|
|
if (this.isRunning) {
|
|
logger.log('Daemon is already running');
|
|
return;
|
|
}
|
|
|
|
logger.log('Starting NUPST daemon...');
|
|
|
|
try {
|
|
// Load configuration - this will throw an error if config doesn't exist
|
|
await this.loadConfig();
|
|
this.logConfigLoaded();
|
|
|
|
// Log version information
|
|
const nupst = this.snmp.getNupst();
|
|
if (nupst) {
|
|
nupst.logVersionInfo(false); // Don't check for updates immediately on startup
|
|
|
|
// Check for updates in the background
|
|
nupst.checkForUpdates().then((updateAvailable: boolean) => {
|
|
if (updateAvailable) {
|
|
const updateStatus = nupst.getUpdateStatus();
|
|
const boxWidth = 45;
|
|
logger.logBoxTitle('Update Available', boxWidth);
|
|
logger.logBoxLine(`Current Version: ${updateStatus.currentVersion}`);
|
|
logger.logBoxLine(`Latest Version: ${updateStatus.latestVersion}`);
|
|
logger.logBoxLine('Run "sudo nupst upgrade" to upgrade');
|
|
logger.logBoxEnd();
|
|
}
|
|
}).catch(() => {}); // Ignore errors checking for updates
|
|
}
|
|
|
|
// Initialize UPS status tracking
|
|
this.initializeUpsStatus();
|
|
|
|
// Start HTTP server if configured
|
|
if (this.config.httpServer?.enabled && this.config.httpServer.authToken) {
|
|
try {
|
|
this.httpServer = new NupstHttpServer(
|
|
this.config.httpServer.port,
|
|
this.config.httpServer.path,
|
|
this.config.httpServer.authToken,
|
|
() => this.upsStatus,
|
|
() => this.pauseState,
|
|
);
|
|
this.httpServer.start();
|
|
} catch (error) {
|
|
logger.error(
|
|
`Failed to start HTTP server: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// Start UPS monitoring
|
|
this.isRunning = true;
|
|
await this.monitor();
|
|
} catch (error) {
|
|
this.isRunning = false;
|
|
logger.error(
|
|
`Daemon failed to start: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
process.exit(1); // Exit with error
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Initialize UPS status tracking for all UPS devices
|
|
*/
|
|
private initializeUpsStatus(): void {
|
|
this.upsStatus.clear();
|
|
|
|
if (this.config.upsDevices && this.config.upsDevices.length > 0) {
|
|
for (const ups of this.config.upsDevices) {
|
|
this.upsStatus.set(ups.id, createInitialUpsStatus(ups));
|
|
}
|
|
|
|
logger.log(`Initialized status tracking for ${this.config.upsDevices.length} UPS devices`);
|
|
} else {
|
|
logger.error('No UPS devices found in configuration');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Log the loaded configuration settings
|
|
*/
|
|
private logConfigLoaded(): void {
|
|
logger.log('');
|
|
logger.logBoxTitle('Configuration Loaded', 70, 'success');
|
|
logger.logBoxLine(`Check Interval: ${this.config.checkInterval / 1000} seconds`);
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Display UPS devices in a table
|
|
if (this.config.upsDevices && this.config.upsDevices.length > 0) {
|
|
logger.info(`UPS Devices (${this.config.upsDevices.length}):`);
|
|
|
|
const upsColumns: Array<
|
|
{ header: string; key: string; align?: 'left' | 'right'; color?: (val: string) => string }
|
|
> = [
|
|
{ header: 'Name', key: 'name', align: 'left', color: theme.highlight },
|
|
{ header: 'ID', key: 'id', align: 'left', color: theme.dim },
|
|
{ header: 'Protocol', key: 'protocol', align: 'left' },
|
|
{ header: 'Host:Port', key: 'host', align: 'left', color: theme.info },
|
|
{ header: 'Actions', key: 'actions', align: 'left' },
|
|
];
|
|
|
|
const upsRows: Array<Record<string, string>> = this.config.upsDevices.map((ups) => {
|
|
const protocol = ups.protocol || 'snmp';
|
|
let host = 'N/A';
|
|
if (protocol === 'upsd' && ups.upsd) {
|
|
host = `${ups.upsd.host}:${ups.upsd.port}`;
|
|
} else if (ups.snmp) {
|
|
host = `${ups.snmp.host}:${ups.snmp.port}`;
|
|
}
|
|
return {
|
|
name: ups.name,
|
|
id: ups.id,
|
|
protocol: protocol.toUpperCase(),
|
|
host,
|
|
actions: `${(ups.actions || []).length} configured`,
|
|
};
|
|
});
|
|
|
|
logger.logTable(upsColumns, upsRows);
|
|
logger.log('');
|
|
} else {
|
|
logger.warn('No UPS devices configured');
|
|
logger.log('');
|
|
}
|
|
|
|
// Display groups in a table
|
|
if (this.config.groups && this.config.groups.length > 0) {
|
|
logger.info(`Groups (${this.config.groups.length}):`);
|
|
|
|
const groupColumns: Array<
|
|
{ header: string; key: string; align?: 'left' | 'right'; color?: (val: string) => string }
|
|
> = [
|
|
{ header: 'Name', key: 'name', align: 'left', color: theme.highlight },
|
|
{ header: 'ID', key: 'id', align: 'left', color: theme.dim },
|
|
{ header: 'Mode', key: 'mode', align: 'left', color: theme.info },
|
|
];
|
|
|
|
const groupRows: Array<Record<string, string>> = this.config.groups.map((group) => ({
|
|
name: group.name,
|
|
id: group.id,
|
|
mode: group.mode,
|
|
}));
|
|
|
|
logger.logTable(groupColumns, groupRows);
|
|
logger.log('');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop the monitoring daemon
|
|
*/
|
|
public stop(): void {
|
|
logger.log('Stopping NUPST daemon...');
|
|
|
|
// Stop HTTP server if running
|
|
if (this.httpServer) {
|
|
this.httpServer.stop();
|
|
}
|
|
|
|
this.isRunning = false;
|
|
}
|
|
|
|
/**
|
|
* Get the current pause state
|
|
*/
|
|
public getPauseState(): IPauseState | null {
|
|
return this.pauseState;
|
|
}
|
|
|
|
/**
|
|
* Check and update pause state from the pause file
|
|
*/
|
|
private checkPauseState(): void {
|
|
const snapshot = loadPauseSnapshot(PAUSE.FILE_PATH, this.isPaused);
|
|
|
|
if (snapshot.transition === 'autoResumed') {
|
|
logger.log('');
|
|
logger.logBoxTitle('Auto-Resume', 45, 'success');
|
|
logger.logBoxLine('Pause duration expired, resuming action monitoring');
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
} else if (snapshot.transition === 'paused' && snapshot.pauseState) {
|
|
logger.log('');
|
|
logger.logBoxTitle('Actions Paused', 45, 'warning');
|
|
logger.logBoxLine(`Paused by: ${snapshot.pauseState.pausedBy}`);
|
|
if (snapshot.pauseState.reason) {
|
|
logger.logBoxLine(`Reason: ${snapshot.pauseState.reason}`);
|
|
}
|
|
if (snapshot.pauseState.resumeAt) {
|
|
const remaining = Math.round((snapshot.pauseState.resumeAt - Date.now()) / 1000);
|
|
logger.logBoxLine(`Auto-resume in: ${remaining} seconds`);
|
|
} else {
|
|
logger.logBoxLine('Duration: Indefinite (run "nupst resume" to resume)');
|
|
}
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
} else if (snapshot.transition === 'resumed') {
|
|
logger.log('');
|
|
logger.logBoxTitle('Actions Resumed', 45, 'success');
|
|
logger.logBoxLine('Action monitoring has been resumed');
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
}
|
|
|
|
this.isPaused = snapshot.isPaused;
|
|
this.pauseState = snapshot.pauseState;
|
|
}
|
|
|
|
/**
|
|
* Monitor the UPS status and trigger shutdown when necessary
|
|
*/
|
|
private async monitor(): Promise<void> {
|
|
logger.log('Starting UPS monitoring...');
|
|
|
|
if (!this.config.upsDevices || this.config.upsDevices.length === 0) {
|
|
logger.warn('No UPS devices found in configuration. Daemon will remain idle...');
|
|
// Don't exit - enter idle monitoring mode instead
|
|
await this.idleMonitoring();
|
|
return;
|
|
}
|
|
|
|
let lastLogTime = 0; // Track when we last logged status
|
|
|
|
// Monitor continuously
|
|
while (this.isRunning) {
|
|
try {
|
|
// Check pause state before each cycle
|
|
this.checkPauseState();
|
|
|
|
// Check all UPS devices (polling continues even when paused for visibility)
|
|
await this.checkAllUpsDevices();
|
|
|
|
// Log periodic status update
|
|
const currentTime = Date.now();
|
|
if (currentTime - lastLogTime >= TIMING.LOG_INTERVAL_MS) {
|
|
this.logAllUpsStatus();
|
|
lastLogTime = currentTime;
|
|
}
|
|
|
|
// Wait before next check
|
|
await this.sleep(this.config.checkInterval);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error during UPS monitoring: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
await this.sleep(this.config.checkInterval);
|
|
}
|
|
}
|
|
|
|
logger.log('UPS monitoring stopped');
|
|
}
|
|
|
|
/**
|
|
* Check status of all UPS devices
|
|
*/
|
|
private async checkAllUpsDevices(): Promise<void> {
|
|
for (const ups of this.config.upsDevices) {
|
|
try {
|
|
const initialStatus = ensureUpsStatus(this.upsStatus.get(ups.id), ups);
|
|
this.upsStatus.set(ups.id, initialStatus);
|
|
|
|
// Check UPS status via configured protocol
|
|
const protocol = ups.protocol || 'snmp';
|
|
const status = protocol === 'upsd' && ups.upsd
|
|
? await this.protocolResolver.getUpsStatus('upsd', undefined, ups.upsd)
|
|
: await this.protocolResolver.getUpsStatus('snmp', ups.snmp);
|
|
const currentTime = Date.now();
|
|
|
|
const currentStatus = this.upsStatus.get(ups.id);
|
|
const pollSnapshot = buildSuccessfulUpsPollSnapshot(
|
|
ups,
|
|
status,
|
|
currentStatus,
|
|
currentTime,
|
|
);
|
|
|
|
if (pollSnapshot.transition === 'recovered' && pollSnapshot.previousStatus) {
|
|
logger.log('');
|
|
logger.logBoxTitle(`UPS Recovered: ${ups.name}`, 60, 'success');
|
|
logger.logBoxLine(`UPS is reachable again after ${pollSnapshot.downtimeSeconds} seconds`);
|
|
logger.logBoxLine(`Current Status: ${formatPowerStatus(status.powerStatus)}`);
|
|
logger.logBoxLine(`Time: ${new Date().toISOString()}`);
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Trigger power status change action for recovery
|
|
await this.triggerUpsActions(
|
|
ups,
|
|
pollSnapshot.updatedStatus,
|
|
pollSnapshot.previousStatus,
|
|
'powerStatusChange',
|
|
);
|
|
} else if (pollSnapshot.transition === 'powerStatusChange' && pollSnapshot.previousStatus) {
|
|
logger.log('');
|
|
logger.logBoxTitle(`Power Status Change: ${ups.name}`, 60, 'warning');
|
|
logger.logBoxLine(
|
|
`Previous: ${formatPowerStatus(pollSnapshot.previousStatus.powerStatus)}`,
|
|
);
|
|
logger.logBoxLine(`Current: ${formatPowerStatus(status.powerStatus)}`);
|
|
logger.logBoxLine(`Time: ${new Date().toISOString()}`);
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Trigger actions for power status change
|
|
await this.triggerUpsActions(
|
|
ups,
|
|
pollSnapshot.updatedStatus,
|
|
pollSnapshot.previousStatus,
|
|
'powerStatusChange',
|
|
);
|
|
}
|
|
|
|
if (
|
|
hasThresholdViolation(
|
|
status.powerStatus,
|
|
status.batteryCapacity,
|
|
status.batteryRuntime,
|
|
ups.actions,
|
|
)
|
|
) {
|
|
await this.triggerUpsActions(
|
|
ups,
|
|
pollSnapshot.updatedStatus,
|
|
pollSnapshot.previousStatus,
|
|
'thresholdViolation',
|
|
);
|
|
}
|
|
|
|
// Update the status in the map
|
|
this.upsStatus.set(ups.id, pollSnapshot.updatedStatus);
|
|
} catch (error) {
|
|
const currentTime = Date.now();
|
|
const currentStatus = this.upsStatus.get(ups.id);
|
|
const failureSnapshot = buildFailedUpsPollSnapshot(ups, currentStatus, currentTime);
|
|
|
|
logger.error(
|
|
`Error checking UPS ${ups.name} (${ups.id}) [failure ${failureSnapshot.failures}/${NETWORK.CONSECUTIVE_FAILURE_THRESHOLD}]: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
|
|
if (failureSnapshot.transition === 'unreachable' && failureSnapshot.previousStatus) {
|
|
logger.log('');
|
|
logger.logBoxTitle(`UPS Unreachable: ${ups.name}`, 60, 'error');
|
|
logger.logBoxLine(`${failureSnapshot.failures} consecutive communication failures`);
|
|
logger.logBoxLine(
|
|
`Last known status: ${formatPowerStatus(failureSnapshot.previousStatus.powerStatus)}`,
|
|
);
|
|
logger.logBoxLine(`Time: ${new Date().toISOString()}`);
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Trigger power status change action for unreachable
|
|
await this.triggerUpsActions(
|
|
ups,
|
|
failureSnapshot.updatedStatus,
|
|
failureSnapshot.previousStatus,
|
|
'powerStatusChange',
|
|
);
|
|
}
|
|
|
|
this.upsStatus.set(ups.id, failureSnapshot.updatedStatus);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Log status of all UPS devices
|
|
*/
|
|
private logAllUpsStatus(): void {
|
|
const timestamp = new Date().toISOString();
|
|
|
|
logger.log('');
|
|
const pauseLabel = this.isPaused ? ' [PAUSED]' : '';
|
|
logger.logBoxTitle(
|
|
`Periodic Status Update${pauseLabel}`,
|
|
70,
|
|
this.isPaused ? 'warning' : 'info',
|
|
);
|
|
logger.logBoxLine(`Timestamp: ${timestamp}`);
|
|
if (this.isPaused && this.pauseState) {
|
|
logger.logBoxLine(`Actions paused by: ${this.pauseState.pausedBy}`);
|
|
if (this.pauseState.resumeAt) {
|
|
const remaining = Math.round((this.pauseState.resumeAt - Date.now()) / 1000);
|
|
logger.logBoxLine(`Auto-resume in: ${remaining > 0 ? remaining : 0} seconds`);
|
|
}
|
|
}
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Build table data
|
|
const columns: Array<
|
|
{ header: string; key: string; align?: 'left' | 'right'; color?: (val: string) => string }
|
|
> = [
|
|
{ header: 'UPS Name', key: 'name', align: 'left', color: theme.highlight },
|
|
{ header: 'ID', key: 'id', align: 'left', color: theme.dim },
|
|
{ header: 'Power Status', key: 'powerStatus', align: 'left' },
|
|
{ header: 'Battery', key: 'battery', align: 'right' },
|
|
{ header: 'Runtime', key: 'runtime', align: 'right' },
|
|
];
|
|
|
|
const rows: Array<Record<string, string>> = [];
|
|
for (const [id, status] of this.upsStatus.entries()) {
|
|
const batteryColor = getBatteryColor(status.batteryCapacity);
|
|
const runtimeColor = getRuntimeColor(status.batteryRuntime);
|
|
|
|
rows.push({
|
|
name: status.name,
|
|
id: id,
|
|
powerStatus: formatPowerStatus(status.powerStatus),
|
|
battery: batteryColor(status.batteryCapacity + '%'),
|
|
runtime: runtimeColor(status.batteryRuntime + ' min'),
|
|
});
|
|
}
|
|
|
|
logger.logTable(columns, rows);
|
|
logger.log('');
|
|
}
|
|
|
|
/**
|
|
* Trigger actions for a UPS device
|
|
* @param ups UPS configuration
|
|
* @param status Current UPS status
|
|
* @param previousStatus Previous UPS status (for determining previousPowerStatus)
|
|
* @param triggerReason Why actions are being triggered
|
|
*/
|
|
private async triggerUpsActions(
|
|
ups: IUpsConfig,
|
|
status: IUpsStatus,
|
|
previousStatus: IUpsStatus | undefined,
|
|
triggerReason: TUpsTriggerReason,
|
|
): Promise<void> {
|
|
const decision = decideUpsActionExecution(
|
|
this.isPaused,
|
|
ups,
|
|
status,
|
|
previousStatus,
|
|
triggerReason,
|
|
);
|
|
|
|
if (decision.type === 'suppressed') {
|
|
logger.info(decision.message);
|
|
return;
|
|
}
|
|
|
|
if (decision.type === 'legacyShutdown') {
|
|
await this.initiateShutdown(decision.reason);
|
|
return;
|
|
}
|
|
|
|
if (decision.type === 'skip') {
|
|
return;
|
|
}
|
|
|
|
const actions = applyDefaultShutdownDelay(
|
|
decision.actions,
|
|
this.getDefaultShutdownDelayMinutes(),
|
|
);
|
|
|
|
await ActionManager.executeActions(actions, decision.context);
|
|
}
|
|
|
|
/**
|
|
* Initiate system shutdown with UPS monitoring during shutdown
|
|
* @param reason Reason for shutdown
|
|
*/
|
|
public async initiateShutdown(reason: string): Promise<void> {
|
|
logger.log(`Initiating system shutdown due to: ${reason}`);
|
|
|
|
const shutdownDelayMinutes = this.getDefaultShutdownDelayMinutes();
|
|
|
|
try {
|
|
await this.shutdownExecutor.scheduleShutdown(shutdownDelayMinutes);
|
|
logger.log(`Allowing ${shutdownDelayMinutes} minutes for VMs to shut down safely`);
|
|
|
|
// Monitor UPS during shutdown and force immediate shutdown if battery gets too low
|
|
logger.log('Monitoring UPS during shutdown process...');
|
|
await this.monitorDuringShutdown();
|
|
} catch (error) {
|
|
logger.error(`Failed to initiate shutdown: ${error}`);
|
|
|
|
const shutdownTriggered = await this.shutdownExecutor.tryScheduledAlternatives();
|
|
if (!shutdownTriggered) {
|
|
logger.error('All shutdown methods failed');
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Monitor UPS during system shutdown
|
|
* Force immediate shutdown if any UPS gets critically low
|
|
*/
|
|
private async monitorDuringShutdown(): Promise<void> {
|
|
const startTime = Date.now();
|
|
|
|
logger.log('');
|
|
logger.logBoxTitle('Shutdown Monitoring Active', UI.WIDE_BOX_WIDTH, 'warning');
|
|
logger.logBoxLine(
|
|
`Emergency threshold: ${THRESHOLDS.EMERGENCY_RUNTIME_MINUTES} minutes runtime`,
|
|
);
|
|
logger.logBoxLine(`Check interval: ${TIMING.SHUTDOWN_CHECK_INTERVAL_MS / 1000} seconds`);
|
|
logger.logBoxLine(`Max monitoring time: ${TIMING.MAX_SHUTDOWN_MONITORING_MS / 1000} seconds`);
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Continue monitoring until max monitoring time is reached
|
|
while (Date.now() - startTime < TIMING.MAX_SHUTDOWN_MONITORING_MS) {
|
|
try {
|
|
logger.info('Checking UPS status during shutdown...');
|
|
|
|
// Build table for UPS status during shutdown
|
|
const columns: Array<
|
|
{ header: string; key: string; align?: 'left' | 'right'; color?: (val: string) => string }
|
|
> = [
|
|
{ header: 'UPS Name', key: 'name', align: 'left', color: theme.highlight },
|
|
{ header: 'Battery', key: 'battery', align: 'right' },
|
|
{ header: 'Runtime', key: 'runtime', align: 'right' },
|
|
{ header: 'Status', key: 'status', align: 'left' },
|
|
];
|
|
|
|
const rows: Array<Record<string, string>> = [];
|
|
let emergencyUps: { ups: IUpsConfig; status: ISnmpUpsStatus } | null = null;
|
|
|
|
// Check all UPS devices
|
|
for (const ups of this.config.upsDevices) {
|
|
try {
|
|
const protocol = ups.protocol || 'snmp';
|
|
const status = protocol === 'upsd' && ups.upsd
|
|
? await this.protocolResolver.getUpsStatus('upsd', undefined, ups.upsd)
|
|
: await this.protocolResolver.getUpsStatus('snmp', ups.snmp);
|
|
const rowSnapshot = buildShutdownStatusRow(
|
|
ups.name,
|
|
status,
|
|
THRESHOLDS.EMERGENCY_RUNTIME_MINUTES,
|
|
{
|
|
battery: (batteryCapacity) =>
|
|
getBatteryColor(batteryCapacity)(`${batteryCapacity}%`),
|
|
runtime: (batteryRuntime) =>
|
|
getRuntimeColor(batteryRuntime)(`${batteryRuntime} min`),
|
|
ok: theme.success,
|
|
critical: theme.error,
|
|
error: theme.error,
|
|
},
|
|
);
|
|
|
|
rows.push(rowSnapshot.row);
|
|
emergencyUps = selectEmergencyCandidate(
|
|
emergencyUps,
|
|
ups,
|
|
status,
|
|
THRESHOLDS.EMERGENCY_RUNTIME_MINUTES,
|
|
);
|
|
} catch (upsError) {
|
|
rows.push(buildShutdownErrorRow(ups.name, theme.error));
|
|
|
|
logger.error(
|
|
`Error checking UPS ${ups.name} during shutdown: ${
|
|
upsError instanceof Error ? upsError.message : String(upsError)
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// Display the table
|
|
logger.logTable(columns, rows);
|
|
logger.log('');
|
|
|
|
// If emergency detected, trigger immediate shutdown
|
|
if (emergencyUps) {
|
|
logger.log('');
|
|
logger.logBoxTitle('EMERGENCY SHUTDOWN', 60, 'error');
|
|
logger.logBoxLine(
|
|
`UPS ${emergencyUps.ups.name} runtime critically low: ${emergencyUps.status.batteryRuntime} minutes`,
|
|
);
|
|
logger.logBoxLine(`Emergency threshold: ${THRESHOLDS.EMERGENCY_RUNTIME_MINUTES} minutes`);
|
|
logger.logBoxLine('Forcing immediate shutdown!');
|
|
logger.logBoxEnd();
|
|
logger.log('');
|
|
|
|
// Force immediate shutdown
|
|
await this.forceImmediateShutdown();
|
|
return;
|
|
}
|
|
|
|
// Wait before checking again
|
|
await this.sleep(TIMING.SHUTDOWN_CHECK_INTERVAL_MS);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error monitoring UPS during shutdown: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
await this.sleep(TIMING.SHUTDOWN_CHECK_INTERVAL_MS);
|
|
}
|
|
}
|
|
|
|
logger.log('');
|
|
logger.success('UPS monitoring during shutdown completed');
|
|
logger.log('');
|
|
}
|
|
|
|
/**
|
|
* Force an immediate system shutdown
|
|
*/
|
|
private async forceImmediateShutdown(): Promise<void> {
|
|
try {
|
|
await this.shutdownExecutor.forceImmediateShutdown();
|
|
} catch (error) {
|
|
logger.error('Emergency shutdown failed, trying alternative methods...');
|
|
|
|
const shutdownTriggered = await this.shutdownExecutor.tryEmergencyAlternatives();
|
|
if (!shutdownTriggered) {
|
|
logger.error('All emergency shutdown methods failed');
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Idle monitoring loop when no UPS devices are configured
|
|
* Watches for config changes and reloads when detected
|
|
*/
|
|
private async idleMonitoring(): Promise<void> {
|
|
let lastConfigCheck = Date.now();
|
|
|
|
logger.log('Entering idle monitoring mode...');
|
|
logger.log(
|
|
`Daemon will check for config changes every ${TIMING.IDLE_CHECK_INTERVAL_MS / 1000} seconds`,
|
|
);
|
|
|
|
// Start file watcher for hot-reload
|
|
this.watchConfigFile();
|
|
|
|
while (this.isRunning) {
|
|
try {
|
|
const currentTime = Date.now();
|
|
|
|
// Periodically check if config has been updated
|
|
if (currentTime - lastConfigCheck >= TIMING.CONFIG_CHECK_INTERVAL_MS) {
|
|
try {
|
|
// Try to load config
|
|
const newConfig = await this.loadConfig();
|
|
|
|
// Check if we now have UPS devices configured
|
|
if (newConfig.upsDevices && newConfig.upsDevices.length > 0) {
|
|
logger.success('Configuration updated! UPS devices found. Starting monitoring...');
|
|
this.initializeUpsStatus();
|
|
// Exit idle mode and start monitoring
|
|
await this.monitor();
|
|
return;
|
|
}
|
|
} catch (error) {
|
|
// Config still doesn't exist or invalid, continue waiting
|
|
}
|
|
|
|
lastConfigCheck = currentTime;
|
|
}
|
|
|
|
await this.sleep(TIMING.IDLE_CHECK_INTERVAL_MS);
|
|
} catch (error) {
|
|
logger.error(
|
|
`Error during idle monitoring: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
await this.sleep(TIMING.IDLE_CHECK_INTERVAL_MS);
|
|
}
|
|
}
|
|
|
|
logger.log('Idle monitoring stopped');
|
|
}
|
|
|
|
/**
|
|
* Watch config file for changes and reload automatically
|
|
*/
|
|
private watchConfigFile(): void {
|
|
try {
|
|
// Use Deno's file watcher to monitor config file
|
|
const configDir = path.dirname(this.CONFIG_PATH);
|
|
|
|
// Spawn a background watcher (non-blocking)
|
|
(async () => {
|
|
try {
|
|
const watcher = Deno.watchFs(configDir);
|
|
|
|
logger.log('Config file watcher started');
|
|
|
|
for await (const event of watcher) {
|
|
// Respond to modify events on config file
|
|
if (shouldReloadConfig(event)) {
|
|
logger.info('Config file changed, reloading...');
|
|
await this.reloadConfig();
|
|
}
|
|
|
|
// Detect pause file changes
|
|
if (shouldRefreshPauseState(event)) {
|
|
this.checkPauseState();
|
|
}
|
|
|
|
// Stop watching if daemon stopped
|
|
if (!this.isRunning) {
|
|
break;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
// Watcher error - not critical, just log it
|
|
logger.dim(
|
|
`Config watcher stopped: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
})();
|
|
} catch (error) {
|
|
// If we can't start the watcher, just log and continue
|
|
// The periodic check will still work
|
|
logger.dim('Could not start config file watcher, using periodic checks only');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reload configuration and restart monitoring if needed
|
|
*/
|
|
private async reloadConfig(): Promise<void> {
|
|
try {
|
|
const oldDeviceCount = this.config.upsDevices?.length || 0;
|
|
|
|
// Load the new configuration
|
|
await this.loadConfig();
|
|
const newDeviceCount = this.config.upsDevices?.length || 0;
|
|
|
|
const reloadSnapshot = analyzeConfigReload(oldDeviceCount, newDeviceCount);
|
|
logger.success(reloadSnapshot.message);
|
|
|
|
if (reloadSnapshot.shouldLogMonitoringStart) {
|
|
logger.info('Monitoring will start automatically...');
|
|
}
|
|
|
|
if (reloadSnapshot.shouldInitializeUpsStatus) {
|
|
// Reinitialize UPS status tracking
|
|
this.initializeUpsStatus();
|
|
}
|
|
} catch (error) {
|
|
logger.warn(
|
|
`Failed to reload config: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sleep for the specified milliseconds
|
|
*/
|
|
private sleep(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
}
|