feat(monitoring): add edge-triggered threshold handling with group action orchestration and HA-aware Proxmox shutdowns

2026-04-16 02:54:16 +00:00
parent bf4d519428
commit a435bd6fed
13 changed files with 1052 additions and 117 deletions
@@ -24,8 +24,16 @@ import {
 import {
  buildFailedUpsPollSnapshot,
  buildSuccessfulUpsPollSnapshot,
+  getActionThresholdStates,
+  getEnteredThresholdIndexes,
  hasThresholdViolation,
+  isActionThresholdExceeded,
 } from '../ts/ups-monitoring.ts';
+import {
+  buildGroupStatusSnapshot,
+  buildGroupThresholdContextStatus,
+  evaluateGroupActionThreshold,
+} from '../ts/group-monitoring.ts';
 import { createInitialUpsStatus } from '../ts/ups-status.ts';

 import * as qenv from 'npm:@push.rocks/qenv@^6.0.0';
@@ -532,6 +540,209 @@ Deno.test('hasThresholdViolation: only fires on battery when any action threshol
  );
 });

+Deno.test('isActionThresholdExceeded: evaluates a single action threshold on battery only', () => {
+  assertEquals(
+    isActionThresholdExceeded(
+      { type: 'shutdown', thresholds: { battery: 50, runtime: 20 } },
+      'online',
+      40,
+      10,
+    ),
+    false,
+  );
+
+  assertEquals(
+    isActionThresholdExceeded(
+      { type: 'shutdown', thresholds: { battery: 50, runtime: 20 } },
+      'onBattery',
+      40,
+      10,
+    ),
+    true,
+  );
+});
+
+Deno.test('getActionThresholdStates: returns per-action threshold state array', () => {
+  assertEquals(
+    getActionThresholdStates('onBattery', 25, 8, [
+      { type: 'shutdown', thresholds: { battery: 30, runtime: 10 } },
+      { type: 'shutdown', thresholds: { battery: 10, runtime: 5 } },
+      { type: 'webhook' },
+    ]),
+    [true, false, false],
+  );
+});
+
+Deno.test('getEnteredThresholdIndexes: reports only newly-entered thresholds', () => {
+  assertEquals(getEnteredThresholdIndexes(undefined, [false, true, true]), [1, 2]);
+  assertEquals(getEnteredThresholdIndexes([false, true, false], [true, true, false]), [0]);
+  assertEquals(getEnteredThresholdIndexes([true, true], [true, false]), []);
+});
+
+// -----------------------------------------------------------------------------
+// Group Monitoring Tests
+// -----------------------------------------------------------------------------
+
+Deno.test('buildGroupStatusSnapshot: redundant group stays online while one UPS remains online', () => {
+  const snapshot = buildGroupStatusSnapshot(
+    { id: 'group-1', name: 'Group Main' },
+    'redundant',
+    [
+      {
+        ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+        powerStatus: 'onBattery' as const,
+        batteryCapacity: 40,
+        batteryRuntime: 12,
+      },
+      {
+        ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+        powerStatus: 'online' as const,
+        batteryCapacity: 98,
+        batteryRuntime: 999,
+      },
+    ],
+    undefined,
+    5000,
+  );
+
+  assertEquals(snapshot.updatedStatus.powerStatus, 'online');
+  assertEquals(snapshot.transition, 'powerStatusChange');
+});
+
+Deno.test('buildGroupStatusSnapshot: nonRedundant group goes unreachable when any member is unreachable', () => {
+  const snapshot = buildGroupStatusSnapshot(
+    { id: 'group-2', name: 'Group Edge' },
+    'nonRedundant',
+    [
+      {
+        ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+        powerStatus: 'online' as const,
+      },
+      {
+        ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+        powerStatus: 'unreachable' as const,
+        unreachableSince: 2000,
+      },
+    ],
+    {
+      ...createInitialUpsStatus({ id: 'group-2', name: 'Group Edge' }, 1000),
+      powerStatus: 'online' as const,
+    },
+    6000,
+  );
+
+  assertEquals(snapshot.updatedStatus.powerStatus, 'unreachable');
+  assertEquals(snapshot.transition, 'powerStatusChange');
+});
+
+Deno.test('evaluateGroupActionThreshold: redundant mode requires all members to be critical', () => {
+  const evaluation = evaluateGroupActionThreshold(
+    { type: 'shutdown', thresholds: { battery: 50, runtime: 20 } },
+    'redundant',
+    [
+      {
+        ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+        powerStatus: 'onBattery' as const,
+        batteryCapacity: 40,
+        batteryRuntime: 15,
+      },
+      {
+        ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+        powerStatus: 'online' as const,
+        batteryCapacity: 95,
+        batteryRuntime: 999,
+      },
+    ],
+  );
+
+  assertEquals(evaluation.exceedsThreshold, false);
+});
+
+Deno.test('evaluateGroupActionThreshold: nonRedundant mode trips on any critical member', () => {
+  const evaluation = evaluateGroupActionThreshold(
+    { type: 'shutdown', thresholds: { battery: 50, runtime: 20 } },
+    'nonRedundant',
+    [
+      {
+        ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+        powerStatus: 'onBattery' as const,
+        batteryCapacity: 40,
+        batteryRuntime: 15,
+      },
+      {
+        ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+        powerStatus: 'online' as const,
+        batteryCapacity: 95,
+        batteryRuntime: 999,
+      },
+    ],
+  );
+
+  assertEquals(evaluation.exceedsThreshold, true);
+  assertEquals(evaluation.blockedByUnreachable, false);
+});
+
+Deno.test('evaluateGroupActionThreshold: blocks destructive actions when a member is unreachable', () => {
+  const evaluation = evaluateGroupActionThreshold(
+    { type: 'proxmox', thresholds: { battery: 50, runtime: 20 } },
+    'nonRedundant',
+    [
+      {
+        ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+        powerStatus: 'onBattery' as const,
+        batteryCapacity: 25,
+        batteryRuntime: 8,
+      },
+      {
+        ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+        powerStatus: 'unreachable' as const,
+        unreachableSince: 3000,
+      },
+    ],
+  );
+
+  assertEquals(evaluation.exceedsThreshold, true);
+  assertEquals(evaluation.blockedByUnreachable, true);
+});
+
+Deno.test('buildGroupThresholdContextStatus: uses the worst triggering member runtime', () => {
+  const status = buildGroupThresholdContextStatus(
+    { id: 'group-3', name: 'Group Worst' },
+    [
+      {
+        exceedsThreshold: true,
+        blockedByUnreachable: false,
+        representativeStatus: {
+          ...createInitialUpsStatus({ id: 'ups-1', name: 'UPS 1' }, 1000),
+          powerStatus: 'onBattery' as const,
+          batteryCapacity: 30,
+          batteryRuntime: 9,
+        },
+      },
+      {
+        exceedsThreshold: true,
+        blockedByUnreachable: false,
+        representativeStatus: {
+          ...createInitialUpsStatus({ id: 'ups-2', name: 'UPS 2' }, 1000),
+          powerStatus: 'onBattery' as const,
+          batteryCapacity: 20,
+          batteryRuntime: 4,
+        },
+      },
+    ],
+    [0, 1],
+    {
+      ...createInitialUpsStatus({ id: 'group-3', name: 'Group Worst' }, 1000),
+      powerStatus: 'online' as const,
+    },
+    7000,
+  );
+
+  assertEquals(status.powerStatus, 'onBattery');
+  assertEquals(status.batteryCapacity, 20);
+  assertEquals(status.batteryRuntime, 4);
+});
+
 // -----------------------------------------------------------------------------
 // UpsOidSets Tests
 // -----------------------------------------------------------------------------