diff --git a/changelog.md b/changelog.md index e82e829..ff2accb 100644 --- a/changelog.md +++ b/changelog.md @@ -3,6 +3,14 @@ ## Pending +### Features + +- support unknown/unreachable duration thresholds (actions) + - Add optional unknownMinutes thresholds for actions when UPS status remains unknown or unreachable. + - Allow shutdown and Proxmox actions to run on configured unknown-state threshold violations. + - Expose unknown duration and threshold values to script and webhook actions. + - Update CLI and systemd output to configure and display unknown/unreachable thresholds. + ## 2026-05-29 - 5.11.2 ### Fixes diff --git a/test/test.ts b/test/test.ts index c48c5fe..0f5e93a 100644 --- a/test/test.ts +++ b/test/test.ts @@ -19,6 +19,7 @@ import { applyDefaultShutdownDelay, buildUpsActionContext, decideUpsActionExecution, + getUnknownStatusDurationMinutes, } from '../ts/action-orchestration.ts'; import { buildShutdownErrorRow, @@ -43,6 +44,7 @@ import { MigrationV4_2ToV4_3 } from '../ts/migrations/migration-v4.2-to-v4.3.ts' import { MigrationV4_3ToV4_4 } from '../ts/migrations/migration-v4.3-to-v4.4.ts'; import { ActionHandler } from '../ts/cli/action-handler.ts'; import { ProxmoxAction } from '../ts/actions/proxmox-action.ts'; +import { ShutdownAction } from '../ts/actions/shutdown-action.ts'; import { renderUpgradeChangelog } from '../ts/upgrade-changelog.ts'; import * as qenv from 'npm:@push.rocks/qenv@^6.0.0'; @@ -302,12 +304,33 @@ Deno.test('buildUpsActionContext: includes previous power status and timestamp', batteryCapacity: 42, batteryRuntime: 15, previousPowerStatus: 'online', + unknownDurationMinutes: 0, timestamp: 9999, triggerReason: 'thresholdViolation', }, ); }); +Deno.test('getUnknownStatusDurationMinutes: tracks unknown and unreachable durations only', () => { + assertEquals( + getUnknownStatusDurationMinutes({ + ...createInitialUpsStatus({ id: 'ups-1', name: 'Main UPS' }, 1_000), + powerStatus: 'unknown', + lastStatusChange: 1_000, + }, 181_000), + 3, + ); + + assertEquals( + getUnknownStatusDurationMinutes({ + ...createInitialUpsStatus({ id: 'ups-1', name: 'Main UPS' }, 1_000), + powerStatus: 'online', + lastStatusChange: 1_000, + }, 181_000), + 0, + ); +}); + Deno.test('decideUpsActionExecution: suppresses actions while paused', () => { const decision = decideUpsActionExecution( true, @@ -368,6 +391,7 @@ Deno.test('decideUpsActionExecution: returns executable action plan when actions batteryCapacity: 55, batteryRuntime: 18, previousPowerStatus: 'online', + unknownDurationMinutes: 0, timestamp: 9999, triggerReason: 'powerStatusChange', }, @@ -571,6 +595,30 @@ Deno.test('isActionThresholdExceeded: evaluates a single action threshold on bat ); }); +Deno.test('isActionThresholdExceeded: evaluates unknown duration threshold separately', () => { + assertEquals( + isActionThresholdExceeded( + { type: 'shutdown', thresholds: { battery: 50, runtime: 20, unknownMinutes: 10 } }, + 'unreachable', + 100, + 60, + 9.9, + ), + false, + ); + + assertEquals( + isActionThresholdExceeded( + { type: 'shutdown', thresholds: { battery: 50, runtime: 20, unknownMinutes: 10 } }, + 'unknown', + 100, + 60, + 10, + ), + true, + ); +}); + Deno.test('getActionThresholdStates: returns per-action threshold state array', () => { assertEquals( getActionThresholdStates('onBattery', 25, 8, [ @@ -582,6 +630,16 @@ Deno.test('getActionThresholdStates: returns per-action threshold state array', ); }); +Deno.test('getActionThresholdStates: includes unknown duration thresholds', () => { + assertEquals( + getActionThresholdStates('unreachable', 100, 60, [ + { type: 'shutdown', thresholds: { battery: 30, runtime: 10, unknownMinutes: 15 } }, + { type: 'shutdown', thresholds: { battery: 30, runtime: 10, unknownMinutes: 20 } }, + ], 15), + [true, false], + ); +}); + Deno.test('getEnteredThresholdIndexes: reports only newly-entered thresholds', () => { assertEquals(getEnteredThresholdIndexes(undefined, [false, true, true]), [1, 2]); assertEquals(getEnteredThresholdIndexes([false, true, false], [true, true, false]), [0]); @@ -714,6 +772,30 @@ Deno.test('evaluateGroupActionThreshold: blocks destructive actions when a membe assertEquals(evaluation.blockedByUnreachable, true); }); +Deno.test('evaluateGroupActionThreshold: allows configured unknown duration threshold', () => { + const evaluation = evaluateGroupActionThreshold( + { type: 'shutdown', thresholds: { battery: 50, runtime: 20, unknownMinutes: 10 } }, + 'nonRedundant', + [ + { + ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000), + powerStatus: 'unreachable' as const, + lastStatusChange: 1000, + unreachableSince: 1000, + }, + { + ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000), + powerStatus: 'online' as const, + }, + ], + 601000, + ); + + assertEquals(evaluation.exceedsThreshold, true); + assertEquals(evaluation.blockedByUnreachable, false); + assertEquals(evaluation.exceededByUnknownDuration, true); +}); + Deno.test('buildGroupThresholdContextStatus: uses the worst triggering member runtime', () => { const status = buildGroupThresholdContextStatus( { id: 'group-3', name: 'Group Worst' }, @@ -721,6 +803,7 @@ Deno.test('buildGroupThresholdContextStatus: uses the worst triggering member ru { exceedsThreshold: true, blockedByUnreachable: false, + exceededByUnknownDuration: false, representativeStatus: { ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000), powerStatus: 'onBattery' as const, @@ -731,6 +814,7 @@ Deno.test('buildGroupThresholdContextStatus: uses the worst triggering member ru { exceedsThreshold: true, blockedByUnreachable: false, + exceededByUnknownDuration: false, representativeStatus: { ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000), powerStatus: 'onBattery' as const, @@ -934,6 +1018,12 @@ class TestProxmoxAction extends ProxmoxAction { } } +class TestShutdownAction extends ShutdownAction { + public testShouldExecute(context: IActionContext): boolean { + return this.shouldExecute(context); + } +} + function createMockContext(overrides: Partial = {}): IActionContext { return { upsId: 'test-ups', @@ -942,6 +1032,7 @@ function createMockContext(overrides: Partial = {}): IActionCont batteryCapacity: 100, batteryRuntime: 60, previousPowerStatus: 'online', + unknownDurationMinutes: 0, timestamp: Date.now(), triggerReason: 'powerStatusChange', ...overrides, @@ -1120,6 +1211,58 @@ Deno.test('ProxmoxAction.shouldExecute: skips thresholds unless UPS is confirmed ); }); +Deno.test('ProxmoxAction.shouldExecute: allows configured unknown duration threshold', () => { + const action = new TestProxmoxAction({ + type: 'proxmox', + triggerMode: 'onlyThresholds', + thresholds: { battery: 50, runtime: 20, unknownMinutes: 10 }, + }); + + assertEquals( + action.testShouldExecute(createMockContext({ + powerStatus: 'unreachable', + triggerReason: 'thresholdViolation', + unknownDurationMinutes: 9.9, + })), + false, + ); + + assertEquals( + action.testShouldExecute(createMockContext({ + powerStatus: 'unreachable', + triggerReason: 'thresholdViolation', + unknownDurationMinutes: 10, + })), + true, + ); +}); + +Deno.test('ShutdownAction.shouldExecute: allows configured unknown duration threshold', () => { + const action = new TestShutdownAction({ + type: 'shutdown', + triggerMode: 'onlyThresholds', + thresholds: { battery: 50, runtime: 20, unknownMinutes: 10 }, + }); + + assertEquals( + action.testShouldExecute(createMockContext({ + powerStatus: 'unknown', + triggerReason: 'thresholdViolation', + unknownDurationMinutes: 10, + })), + true, + ); + + assertEquals( + action.testShouldExecute(createMockContext({ + powerStatus: 'unknown', + triggerReason: 'powerStatusChange', + unknownDurationMinutes: 10, + })), + false, + ); +}); + Deno.test('ProxmoxAction.shouldExecute: allows confirmed battery events', () => { const powerChangeAction = new TestProxmoxAction({ type: 'proxmox' }); const thresholdAction = new TestProxmoxAction({ @@ -1193,7 +1336,7 @@ Deno.test('ActionHandler.runEditProcess: updates an existing shutdown action', a } as unknown as ConstructorParameters[0]; const handler = new ActionHandler(nupstMock); - const answers = ['', '12', '25', '8', '3']; + const answers = ['', '12', '25', '8', '', '3']; let answerIndex = 0; const prompt = async (_question: string): Promise => answers[answerIndex++] ?? ''; diff --git a/ts/action-orchestration.ts b/ts/action-orchestration.ts index 0059b8f..a27e7df 100644 --- a/ts/action-orchestration.ts +++ b/ts/action-orchestration.ts @@ -15,6 +15,14 @@ export type TActionExecutionDecision = | { type: 'skip' } | { type: 'execute'; actions: IActionConfig[]; context: IActionContext }; +export function getUnknownStatusDurationMinutes(status: IUpsStatus, timestamp: number): number { + if (status.powerStatus !== 'unknown' && status.powerStatus !== 'unreachable') { + return 0; + } + + return Math.max(0, (timestamp - status.lastStatusChange) / 60_000); +} + export function buildUpsActionContext( ups: IUpsActionSource, status: IUpsStatus, @@ -29,6 +37,7 @@ export function buildUpsActionContext( batteryCapacity: status.batteryCapacity, batteryRuntime: status.batteryRuntime, previousPowerStatus: (previousStatus?.powerStatus || 'unknown') as TPowerStatus, + unknownDurationMinutes: getUnknownStatusDurationMinutes(status, timestamp), timestamp, triggerReason, }; diff --git a/ts/actions/base-action.ts b/ts/actions/base-action.ts index 334cd03..46cc3bb 100644 --- a/ts/actions/base-action.ts +++ b/ts/actions/base-action.ts @@ -3,7 +3,7 @@ * * Actions are triggered on: * 1. Power status changes (online ↔ onBattery) - * 2. Threshold violations (battery/runtime cross below configured thresholds) + * 2. Threshold violations (battery/runtime or unknown-state duration cross configured thresholds) */ export type TPowerStatus = 'online' | 'onBattery' | 'unknown' | 'unreachable'; @@ -30,6 +30,8 @@ export interface IActionContext { // State tracking /** Previous power status before this trigger */ previousPowerStatus: TPowerStatus; + /** Minutes the UPS has continuously been unknown/unreachable. Zero for known states. */ + unknownDurationMinutes: number; // Metadata /** Timestamp when this action was triggered (milliseconds since epoch) */ @@ -71,6 +73,8 @@ export interface IActionConfig { battery: number; /** Runtime threshold in minutes */ runtime: number; + /** Optional fail-safe threshold for unknown/unreachable status duration in minutes */ + unknownMinutes?: number; }; // Shutdown action configuration @@ -158,13 +162,13 @@ export abstract class Action { case 'onlyThresholds': // Only execute when this action's thresholds are exceeded if (!this.config.thresholds) return false; // No thresholds = never execute - return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime); + return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime, context); case 'powerChangesAndThresholds': // Execute on power changes OR when thresholds exceeded if (context.triggerReason === 'powerStatusChange') return true; if (!this.config.thresholds) return false; - return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime); + return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime, context); case 'anyChange': // Execute on every trigger (power change or threshold check) @@ -176,19 +180,43 @@ export abstract class Action { } /** - * Check if current battery/runtime exceeds this action's thresholds + * Check if current UPS state exceeds this action's thresholds * @param batteryCapacity Current battery percentage * @param batteryRuntime Current runtime in minutes * @returns True if thresholds are exceeded */ - protected areThresholdsExceeded(batteryCapacity: number, batteryRuntime: number): boolean { + protected areThresholdsExceeded( + batteryCapacity: number, + batteryRuntime: number, + context?: IActionContext, + ): boolean { if (!this.config.thresholds) { return false; // No thresholds configured } + if (context && this.isUnknownThresholdExceeded(context)) { + return true; + } + return ( batteryCapacity < this.config.thresholds.battery || batteryRuntime < this.config.thresholds.runtime ); } + + protected isUnknownThresholdExceeded(context: IActionContext): boolean { + const unknownMinutes = this.config.thresholds?.unknownMinutes; + if ( + typeof unknownMinutes !== 'number' || + !Number.isFinite(unknownMinutes) || + unknownMinutes < 0 + ) { + return false; + } + + return ( + (context.powerStatus === 'unknown' || context.powerStatus === 'unreachable') && + context.unknownDurationMinutes >= unknownMinutes + ); + } } diff --git a/ts/actions/proxmox-action.ts b/ts/actions/proxmox-action.ts index 7e50bc4..a8d1032 100644 --- a/ts/actions/proxmox-action.ts +++ b/ts/actions/proxmox-action.ts @@ -34,6 +34,19 @@ export class ProxmoxAction extends Action { const mode = this.config.triggerMode || 'powerChangesAndThresholds'; if (context.powerStatus !== 'onBattery') { + if ( + context.triggerReason === 'thresholdViolation' && + mode !== 'onlyPowerChanges' && + this.isUnknownThresholdExceeded(context) + ) { + logger.warn( + `Proxmox action triggered: UPS ${context.powerStatus} for ${ + context.unknownDurationMinutes.toFixed(1) + } minutes`, + ); + return true; + } + if (context.powerStatus === 'unreachable') { logger.info( 'Proxmox action skipped: UPS is unreachable (communication failure, actual state unknown)', @@ -52,7 +65,7 @@ export class ProxmoxAction extends Action { return false; } - return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime); + return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime, context); } if (context.triggerReason === 'powerStatusChange') { diff --git a/ts/actions/script-action.ts b/ts/actions/script-action.ts index 28b8a1a..fbe8d1d 100644 --- a/ts/actions/script-action.ts +++ b/ts/actions/script-action.ts @@ -112,11 +112,15 @@ export class ScriptAction extends Action { NUPST_POWER_STATUS: context.powerStatus, NUPST_BATTERY_CAPACITY: String(context.batteryCapacity), NUPST_BATTERY_RUNTIME: String(context.batteryRuntime), + NUPST_UNKNOWN_DURATION_MINUTES: String(context.unknownDurationMinutes), NUPST_TRIGGER_REASON: context.triggerReason, NUPST_TIMESTAMP: String(context.timestamp), // Include action's own thresholds if configured NUPST_BATTERY_THRESHOLD: this.config.thresholds ? String(this.config.thresholds.battery) : '', NUPST_RUNTIME_THRESHOLD: this.config.thresholds ? String(this.config.thresholds.runtime) : '', + NUPST_UNKNOWN_THRESHOLD_MINUTES: this.config.thresholds?.unknownMinutes !== undefined + ? String(this.config.thresholds.unknownMinutes) + : '', }; // Build command with arguments diff --git a/ts/actions/shutdown-action.ts b/ts/actions/shutdown-action.ts index 941e531..210d76b 100644 --- a/ts/actions/shutdown-action.ts +++ b/ts/actions/shutdown-action.ts @@ -35,8 +35,21 @@ export class ShutdownAction extends Action { // CRITICAL SAFETY CHECK: Shutdown should NEVER trigger unless UPS is on battery // A low battery while on grid power is not an emergency (the battery is charging) - // When UPS is unreachable, we don't know the actual state - don't trigger false shutdown + // Unknown/unreachable status can only trigger after an explicit unknownMinutes threshold. if (context.powerStatus !== 'onBattery') { + if ( + context.triggerReason === 'thresholdViolation' && + mode !== 'onlyPowerChanges' && + this.isUnknownThresholdExceeded(context) + ) { + logger.warn( + `Shutdown action triggered: UPS ${context.powerStatus} for ${ + context.unknownDurationMinutes.toFixed(1) + } minutes`, + ); + return true; + } + if (context.powerStatus === 'unreachable') { logger.info( `Shutdown action skipped: UPS is unreachable (communication failure, actual state unknown)`, @@ -57,7 +70,7 @@ export class ShutdownAction extends Action { return false; } // Check if thresholds are actually exceeded - return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime); + return this.areThresholdsExceeded(context.batteryCapacity, context.batteryRuntime, context); } // Handle power status changes diff --git a/ts/actions/webhook-action.ts b/ts/actions/webhook-action.ts index 7829522..a51e2b4 100644 --- a/ts/actions/webhook-action.ts +++ b/ts/actions/webhook-action.ts @@ -19,6 +19,8 @@ export interface IWebhookPayload { batteryCapacity: number; /** Current battery runtime in minutes */ batteryRuntime: number; + /** Minutes the UPS has continuously been unknown/unreachable */ + unknownDurationMinutes: number; /** Reason this webhook was triggered */ triggerReason: 'powerStatusChange' | 'thresholdViolation'; /** Timestamp when webhook was triggered */ @@ -27,6 +29,7 @@ export interface IWebhookPayload { thresholds?: { battery: number; runtime: number; + unknownMinutes?: number; }; } @@ -92,6 +95,7 @@ export class WebhookAction extends Action { powerStatus: context.powerStatus, batteryCapacity: context.batteryCapacity, batteryRuntime: context.batteryRuntime, + unknownDurationMinutes: context.unknownDurationMinutes, triggerReason: context.triggerReason, timestamp: context.timestamp, }; @@ -101,6 +105,9 @@ export class WebhookAction extends Action { payload.thresholds = { battery: this.config.thresholds.battery, runtime: this.config.thresholds.runtime, + ...(this.config.thresholds.unknownMinutes !== undefined + ? { unknownMinutes: this.config.thresholds.unknownMinutes } + : {}), }; } @@ -113,6 +120,7 @@ export class WebhookAction extends Action { url.searchParams.append('powerStatus', payload.powerStatus); url.searchParams.append('batteryCapacity', String(payload.batteryCapacity)); url.searchParams.append('batteryRuntime', String(payload.batteryRuntime)); + url.searchParams.append('unknownDurationMinutes', String(context.unknownDurationMinutes)); url.searchParams.append('triggerReason', payload.triggerReason); url.searchParams.append('timestamp', String(payload.timestamp)); diff --git a/ts/cli/action-handler.ts b/ts/cli/action-handler.ts index 1a65484..4120734 100644 --- a/ts/cli/action-handler.ts +++ b/ts/cli/action-handler.ts @@ -242,10 +242,13 @@ export class ActionHandler { logger.success(`Action removed from ${targetType} ${targetName}`); logger.log(` ${theme.dim('Type:')} ${removedAction.type}`); if (removedAction.thresholds) { + const unknownThreshold = removedAction.thresholds.unknownMinutes !== undefined + ? `, Unknown: ${removedAction.thresholds.unknownMinutes}min` + : ''; logger.log( ` ${ theme.dim('Thresholds:') - } Battery: ${removedAction.thresholds.battery}%, Runtime: ${removedAction.thresholds.runtime}min`, + } Battery: ${removedAction.thresholds.battery}%, Runtime: ${removedAction.thresholds.runtime}min${unknownThreshold}`, ); } logger.log(` ${theme.dim('Changes saved and will be applied automatically')}`); @@ -703,7 +706,31 @@ export class ActionHandler { logger.error('Invalid runtime threshold. Must be >= 0.'); process.exit(1); } - newAction.thresholds = { battery, runtime }; + const thresholds: NonNullable = { battery, runtime }; + + const currentUnknownThreshold = existingAction?.thresholds?.unknownMinutes; + const unknownPrompt = currentUnknownThreshold !== undefined + ? ` ${theme.dim('Unknown/unreachable threshold')} ${ + theme.dim(`(minutes, 'clear' = disabled) [${currentUnknownThreshold}]:`) + } ` + : ` ${theme.dim('Unknown/unreachable threshold')} ${ + theme.dim('(minutes, empty = disabled):') + } `; + const unknownInput = await prompt(unknownPrompt); + if (this.isClearInput(unknownInput)) { + // Leave disabled. + } else if (unknownInput.trim()) { + const unknownMinutes = parseInt(unknownInput, 10); + if (isNaN(unknownMinutes) || unknownMinutes < 0) { + logger.error('Invalid unknown/unreachable threshold. Must be >= 0.'); + process.exit(1); + } + thresholds.unknownMinutes = unknownMinutes; + } else if (currentUnknownThreshold !== undefined) { + thresholds.unknownMinutes = currentUnknownThreshold; + } + + newAction.thresholds = thresholds; logger.log(''); logger.log(` ${theme.dim('Trigger mode:')}`); @@ -758,6 +785,7 @@ export class ActionHandler { { header: 'Type', key: 'type', align: 'left' }, { header: 'Battery', key: 'battery', align: 'right' }, { header: 'Runtime', key: 'runtime', align: 'right' }, + { header: 'Unknown', key: 'unknown', align: 'right' }, { header: 'Trigger Mode', key: 'triggerMode', align: 'left' }, { header: 'Details', key: 'details', align: 'left' }, ]; @@ -792,6 +820,9 @@ export class ActionHandler { type: theme.highlight(action.type), battery: action.thresholds ? `${action.thresholds.battery}%` : theme.dim('N/A'), runtime: action.thresholds ? `${action.thresholds.runtime}min` : theme.dim('N/A'), + unknown: action.thresholds?.unknownMinutes !== undefined + ? `${action.thresholds.unknownMinutes}min` + : theme.dim('off'), triggerMode: theme.dim(action.triggerMode || 'onlyThresholds'), details, }; diff --git a/ts/cli/ups-handler.ts b/ts/cli/ups-handler.ts index 305c160..2a4ef67 100644 --- a/ts/cli/ups-handler.ts +++ b/ts/cli/ups-handler.ts @@ -1318,7 +1318,10 @@ export class UpsHandler { logger.log(''); logger.info('Action Thresholds:'); logger.dim( - 'Action will trigger when battery or runtime falls below these values (while on battery)', + 'Action will trigger when battery/runtime falls below these values while on battery.', + ); + logger.dim( + 'Optionally, it can also trigger after the UPS has been unknown/unreachable for N minutes.', ); const batteryInput = await prompt('Battery threshold percentage [60]: '); @@ -1329,10 +1332,18 @@ export class UpsHandler { const runtime = parseInt(runtimeInput, 10); const runtimeThreshold = (runtimeInput.trim() && !isNaN(runtime)) ? runtime : 20; + const unknownInput = await prompt( + 'Unknown/unreachable threshold in minutes (empty = disabled): ', + ); + const unknownMinutes = parseInt(unknownInput, 10); + action.thresholds = { battery: batteryThreshold, runtime: runtimeThreshold, }; + if (unknownInput.trim() && !isNaN(unknownMinutes) && unknownMinutes >= 0) { + action.thresholds.unknownMinutes = unknownMinutes; + } } actions.push(action as IActionConfig); diff --git a/ts/daemon.ts b/ts/daemon.ts index 35f888b..97c5424 100644 --- a/ts/daemon.ts +++ b/ts/daemon.ts @@ -38,6 +38,7 @@ import { ensureUpsStatus, getActionThresholdStates, getEnteredThresholdIndexes, + getUnknownStatusDurationMinutes, } from './ups-monitoring.ts'; import { buildShutdownErrorRow, @@ -653,10 +654,11 @@ export class NupstDaemon { } const thresholdStates = getActionThresholdStates( - status.powerStatus, - status.batteryCapacity, - status.batteryRuntime, + pollSnapshot.updatedStatus.powerStatus, + pollSnapshot.updatedStatus.batteryCapacity, + pollSnapshot.updatedStatus.batteryRuntime, ups.actions, + getUnknownStatusDurationMinutes(pollSnapshot.updatedStatus, currentTime), ); const enteredThresholdIndexes = this.trackEnteredThresholdIndexes( `ups:${ups.id}`, @@ -706,6 +708,28 @@ export class NupstDaemon { ); } + const thresholdStates = getActionThresholdStates( + failureSnapshot.updatedStatus.powerStatus, + failureSnapshot.updatedStatus.batteryCapacity, + failureSnapshot.updatedStatus.batteryRuntime, + ups.actions, + getUnknownStatusDurationMinutes(failureSnapshot.updatedStatus, currentTime), + ); + const enteredThresholdIndexes = this.trackEnteredThresholdIndexes( + `ups:${ups.id}`, + thresholdStates, + ); + + if (enteredThresholdIndexes.length > 0) { + await this.triggerUpsActions( + ups, + failureSnapshot.updatedStatus, + failureSnapshot.previousStatus, + 'thresholdViolation', + enteredThresholdIndexes, + ); + } + this.upsStatus.set(ups.id, failureSnapshot.updatedStatus); } } @@ -769,7 +793,7 @@ export class NupstDaemon { } const thresholdEvaluations = (group.actions || []).map((action) => - evaluateGroupActionThreshold(action, group.mode, memberStatuses) + evaluateGroupActionThreshold(action, group.mode, memberStatuses, currentTime) ); const thresholdStates = thresholdEvaluations.map((evaluation) => evaluation.exceedsThreshold && !evaluation.blockedByUnreachable diff --git a/ts/group-monitoring.ts b/ts/group-monitoring.ts index 57f7c68..e8e75b1 100644 --- a/ts/group-monitoring.ts +++ b/ts/group-monitoring.ts @@ -1,4 +1,5 @@ import type { IActionConfig, TPowerStatus } from './actions/base-action.ts'; +import { getUnknownStatusDurationMinutes, isActionThresholdExceeded } from './ups-monitoring.ts'; import { createInitialUpsStatus, type IUpsIdentity, type IUpsStatus } from './ups-status.ts'; export interface IGroupStatusSnapshot { @@ -10,6 +11,7 @@ export interface IGroupStatusSnapshot { export interface IGroupThresholdEvaluation { exceedsThreshold: boolean; blockedByUnreachable: boolean; + exceededByUnknownDuration: boolean; representativeStatus?: IUpsStatus; } @@ -143,11 +145,13 @@ export function evaluateGroupActionThreshold( actionConfig: IActionConfig, mode: 'redundant' | 'nonRedundant', memberStatuses: IUpsStatus[], + currentTime: number = Date.now(), ): IGroupThresholdEvaluation { if (!actionConfig.thresholds || memberStatuses.length === 0) { return { exceedsThreshold: false, blockedByUnreachable: false, + exceededByUnknownDuration: false, }; } @@ -156,16 +160,32 @@ export function evaluateGroupActionThreshold( (status.batteryCapacity < actionConfig.thresholds!.battery || status.batteryRuntime < actionConfig.thresholds!.runtime) ); - const exceedsThreshold = mode === 'redundant' + + const unknownMembers = memberStatuses.filter((status) => + isActionThresholdExceeded( + actionConfig, + status.powerStatus, + status.batteryCapacity, + status.batteryRuntime, + getUnknownStatusDurationMinutes(status, currentTime), + ) && (status.powerStatus === 'unknown' || status.powerStatus === 'unreachable') + ); + + const exceedsBatteryThreshold = mode === 'redundant' ? criticalMembers.length === memberStatuses.length : criticalMembers.length > 0; + const exceededByUnknownDuration = mode === 'redundant' + ? unknownMembers.length === memberStatuses.length + : unknownMembers.length > 0; + const exceedsThreshold = exceedsBatteryThreshold || exceededByUnknownDuration; return { exceedsThreshold, - blockedByUnreachable: exceedsThreshold && + blockedByUnreachable: exceedsBatteryThreshold && !exceededByUnknownDuration && destructiveActionTypes.has(actionConfig.type) && memberStatuses.some((status) => status.powerStatus === 'unreachable'), - representativeStatus: selectWorstStatus(criticalMembers), + exceededByUnknownDuration, + representativeStatus: selectWorstStatus([...criticalMembers, ...unknownMembers]), }; } @@ -181,18 +201,23 @@ export function buildGroupThresholdContextStatus( .filter((status): status is IUpsStatus => !!status); const representative = selectWorstStatus(representativeStatuses) || fallbackStatus; + const powerStatus = representative.powerStatus === 'unknown' || + representative.powerStatus === 'unreachable' + ? representative.powerStatus + : 'onBattery'; return { ...fallbackStatus, id: group.id, name: group.name, - powerStatus: 'onBattery', + powerStatus, batteryCapacity: representative.batteryCapacity, batteryRuntime: representative.batteryRuntime, outputLoad: representative.outputLoad, outputPower: representative.outputPower, outputVoltage: representative.outputVoltage, outputCurrent: representative.outputCurrent, + lastStatusChange: representative.lastStatusChange, lastCheckTime: currentTime, }; } diff --git a/ts/systemd.ts b/ts/systemd.ts index faca645..d627c5b 100644 --- a/ts/systemd.ts +++ b/ts/systemd.ts @@ -512,6 +512,9 @@ WantedBy=multi-user.target actionDesc += ` (${ action.triggerMode || 'onlyThresholds' }: battery<${action.thresholds.battery}%, runtime<${action.thresholds.runtime}min`; + if (action.thresholds.unknownMinutes !== undefined) { + actionDesc += `, unknown>${action.thresholds.unknownMinutes}min`; + } if (action.type === 'shutdown') { const shutdownDelay = action.shutdownDelay ?? defaultShutdownDelay; actionDesc += `, delay=${shutdownDelay}min`; @@ -599,6 +602,9 @@ WantedBy=multi-user.target actionDesc += ` (${ action.triggerMode || 'onlyThresholds' }: battery<${action.thresholds.battery}%, runtime<${action.thresholds.runtime}min`; + if (action.thresholds.unknownMinutes !== undefined) { + actionDesc += `, unknown>${action.thresholds.unknownMinutes}min`; + } if (action.type === 'shutdown') { const shutdownDelay = action.shutdownDelay ?? defaultShutdownDelay; actionDesc += `, delay=${shutdownDelay}min`; diff --git a/ts/ups-monitoring.ts b/ts/ups-monitoring.ts index e9583ce..19b7f85 100644 --- a/ts/ups-monitoring.ts +++ b/ts/ups-monitoring.ts @@ -1,4 +1,5 @@ import type { IActionConfig } from './actions/base-action.ts'; +import { getUnknownStatusDurationMinutes } from './action-orchestration.ts'; import { NETWORK } from './constants.ts'; import type { IUpsStatus as IProtocolUpsStatus } from './snmp/types.ts'; import { createInitialUpsStatus, type IUpsIdentity, type IUpsStatus } from './ups-status.ts'; @@ -119,8 +120,15 @@ export function hasThresholdViolation( batteryCapacity: number, batteryRuntime: number, actions: IActionConfig[] | undefined, + unknownDurationMinutes: number = 0, ): boolean { - return getActionThresholdStates(powerStatus, batteryCapacity, batteryRuntime, actions).some( + return getActionThresholdStates( + powerStatus, + batteryCapacity, + batteryRuntime, + actions, + unknownDurationMinutes, + ).some( Boolean, ); } @@ -130,8 +138,23 @@ export function isActionThresholdExceeded( powerStatus: IProtocolUpsStatus['powerStatus'], batteryCapacity: number, batteryRuntime: number, + unknownDurationMinutes: number = 0, ): boolean { - if (powerStatus !== 'onBattery' || !actionConfig.thresholds) { + if (!actionConfig.thresholds) { + return false; + } + + const unknownMinutes = actionConfig.thresholds.unknownMinutes; + if ( + (powerStatus === 'unknown' || powerStatus === 'unreachable') && + typeof unknownMinutes === 'number' && + Number.isFinite(unknownMinutes) && + unknownMinutes >= 0 + ) { + return unknownDurationMinutes >= unknownMinutes; + } + + if (powerStatus !== 'onBattery') { return false; } @@ -146,16 +169,25 @@ export function getActionThresholdStates( batteryCapacity: number, batteryRuntime: number, actions: IActionConfig[] | undefined, + unknownDurationMinutes: number = 0, ): boolean[] { if (!actions || actions.length === 0) { return []; } return actions.map((actionConfig) => - isActionThresholdExceeded(actionConfig, powerStatus, batteryCapacity, batteryRuntime) + isActionThresholdExceeded( + actionConfig, + powerStatus, + batteryCapacity, + batteryRuntime, + unknownDurationMinutes, + ) ); } +export { getUnknownStatusDurationMinutes }; + export function getEnteredThresholdIndexes( previousStates: boolean[] | undefined, currentStates: boolean[],