feat(opsserver): add health, audit, cluster health, and durable credential management hardening

test/test.cluster-health.test.ts

@@ -0,0 +1,67 @@
+import { assertEquals } from 'jsr:@std/assert';
+import { describe, it } from 'jsr:@std/testing/bdd';
+import { TypedRequest } from '@api.global/typedrequest';
+import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helpers/server.helper.ts';
+import { ObjectStorageContainer } from '../ts/index.ts';
+import type { IReq_GetClusterHealth } from '../ts_interfaces/requests/status.ts';
+import type * as interfaces from '../ts_interfaces/index.ts';
+
+const PORT_INDEX = 9;
+const ports = getTestPorts(PORT_INDEX);
+const url = `http://localhost:${ports.uiPort}/typedrequest`;
+const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
+
+const cleanupStorageDirectory = async () => {
+  try {
+    await Deno.remove(storageDirectory, { recursive: true });
+  } catch (error) {
+    if (!(error instanceof Deno.errors.NotFound)) {
+      throw error;
+    }
+  }
+};
+
+describe('Cluster health', { sanitizeResources: false, sanitizeOps: false }, () => {
+  it('exposes smartstorage cluster health through the management API', async () => {
+    await cleanupStorageDirectory();
+
+    let container: ObjectStorageContainer | null = null;
+    try {
+      const drivePaths = Array.from({ length: 6 }, (_value, index) => {
+        return `${storageDirectory}/drive-${index + 1}`;
+      });
+
+      container = createTestContainer(PORT_INDEX, {
+        storageDirectory,
+        clusterEnabled: true,
+        clusterNodeId: 'objectstorage-test-node',
+        clusterQuicPort: 19433,
+        drivePaths,
+        erasureDataShards: 4,
+        erasureParityShards: 2,
+        erasureChunkSizeBytes: 1024 * 1024,
+      });
+      await container.start();
+
+      const identity: interfaces.data.IIdentity = await loginAndGetIdentity(ports.uiPort);
+      const req = new TypedRequest<IReq_GetClusterHealth>(url, 'getClusterHealth');
+      const response = await req.fire({ identity });
+      const health = response.clusterHealth;
+
+      assertEquals(health.enabled, true);
+      assertEquals(health.nodeId, 'objectstorage-test-node');
+      assertEquals(health.quorumHealthy, true);
+      assertEquals(health.majorityHealthy, true);
+      assertEquals(health.drives?.length, 6);
+      assertEquals(health.drives?.every((drive) => drive.status === 'online'), true);
+      assertEquals(health.erasure?.dataShards, 4);
+      assertEquals(health.erasure?.parityShards, 2);
+      assertEquals(health.erasure?.totalShards, 6);
+    } finally {
+      if (container) {
+        await container.stop();
+      }
+      await cleanupStorageDirectory();
+    }
+  });
+});

test/test.credentials-persistence.test.ts

@@ -0,0 +1,206 @@
+import { assertEquals, assertRejects } from 'jsr:@std/assert';
+import { describe, it } from 'jsr:@std/testing/bdd';
+import { TypedRequest } from '@api.global/typedrequest';
+import {
+  createTestContainer,
+  getTestPorts,
+  loginAndGetIdentity,
+  TEST_ACCESS_KEY,
+} from './helpers/server.helper.ts';
+import { ObjectStorageContainer } from '../ts/index.ts';
+import type { IReq_CreateBucket, IReq_ListBuckets } from '../ts_interfaces/requests/buckets.ts';
+import type {
+  IReq_AddCredential,
+  IReq_GetCredentials,
+  IReq_RemoveCredential,
+} from '../ts_interfaces/requests/credentials.ts';
+import type * as interfaces from '../ts_interfaces/index.ts';
+
+const PORT_INDEX = 8;
+const ports = getTestPorts(PORT_INDEX);
+const url = `http://localhost:${ports.uiPort}/typedrequest`;
+const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
+
+const cleanupStorageDirectory = async () => {
+  try {
+    await Deno.remove(storageDirectory, { recursive: true });
+  } catch (error) {
+    if (!(error instanceof Deno.errors.NotFound)) {
+      throw error;
+    }
+  }
+};
+
+describe('Credential persistence', { sanitizeResources: false, sanitizeOps: false }, () => {
+  it('persists managed credentials across restart and refreshes the internal client', async () => {
+    await cleanupStorageDirectory();
+
+    let activeContainer: ObjectStorageContainer | null = null;
+
+    const stopContainer = async () => {
+      if (!activeContainer) {
+        return;
+      }
+
+      try {
+        await activeContainer.stop();
+      } finally {
+        activeContainer = null;
+      }
+    };
+
+    try {
+      activeContainer = createTestContainer(PORT_INDEX);
+      await activeContainer.start();
+
+      let identity: interfaces.data.IIdentity = await loginAndGetIdentity(ports.uiPort);
+
+      const addCredential = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
+      await addCredential.fire({
+        identity,
+        accessKeyId: 'persisted-key',
+        secretAccessKey: 'persisted-secret',
+      });
+
+      const removeCredential = new TypedRequest<IReq_RemoveCredential>(url, 'removeCredential');
+      await removeCredential.fire({ identity, accessKeyId: TEST_ACCESS_KEY });
+
+      const getCredentials = new TypedRequest<IReq_GetCredentials>(url, 'getCredentials');
+      const credentialsBeforeRestart = await getCredentials.fire({ identity });
+      assertEquals(credentialsBeforeRestart.credentials.length, 1);
+      assertEquals(credentialsBeforeRestart.credentials[0].accessKeyId, 'persisted-key');
+
+      const listBuckets = new TypedRequest<IReq_ListBuckets>(url, 'listBuckets');
+      const bucketsBeforeRestart = await listBuckets.fire({ identity });
+      assertEquals(Array.isArray(bucketsBeforeRestart.buckets), true);
+
+      await stopContainer();
+
+      activeContainer = createTestContainer(PORT_INDEX);
+      await activeContainer.start();
+
+      identity = await loginAndGetIdentity(ports.uiPort);
+
+      const credentialsAfterRestart = await getCredentials.fire({ identity });
+      assertEquals(credentialsAfterRestart.credentials.length, 1);
+      assertEquals(credentialsAfterRestart.credentials[0].accessKeyId, 'persisted-key');
+
+      const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
+      await createBucket.fire({ identity, bucketName: 'persisted-creds-bucket' });
+
+      const bucketsAfterRestart = await listBuckets.fire({ identity });
+      assertEquals(
+        bucketsAfterRestart.buckets.some((bucket) => bucket.name === 'persisted-creds-bucket'),
+        true,
+      );
+    } finally {
+      await stopContainer();
+      await cleanupStorageDirectory();
+    }
+  });
+
+  it('lets explicit environment credentials override persisted managed credentials', async () => {
+    const portIndex = 10;
+    const envPorts = getTestPorts(portIndex);
+    const envUrl = `http://localhost:${envPorts.uiPort}/typedrequest`;
+    const envStorageDirectory = `.nogit/testdata-${portIndex}`;
+    const previousAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
+    const previousSecretKey = Deno.env.get('OBJST_SECRET_KEY');
+    let container: ObjectStorageContainer | null = null;
+
+    const cleanupEnvStorageDirectory = async () => {
+      try {
+        await Deno.remove(envStorageDirectory, { recursive: true });
+      } catch (error) {
+        if (!(error instanceof Deno.errors.NotFound)) {
+          throw error;
+        }
+      }
+    };
+
+    try {
+      await cleanupEnvStorageDirectory();
+      await Deno.mkdir(`${envStorageDirectory}/.objectstorage`, { recursive: true });
+      await Deno.writeTextFile(
+        `${envStorageDirectory}/.objectstorage/admin-config.json`,
+        JSON.stringify({
+          accessCredentials: [{ accessKeyId: 'persisted-key', secretAccessKey: 'persisted-secret' }],
+        }),
+      );
+
+      Deno.env.set('OBJST_ACCESS_KEY', 'env-key');
+      Deno.env.set('OBJST_SECRET_KEY', 'env-secret');
+
+      container = createTestContainer(portIndex, { storageDirectory: envStorageDirectory });
+      await container.start();
+
+      const identity = await loginAndGetIdentity(envPorts.uiPort);
+      const getCredentials = new TypedRequest<IReq_GetCredentials>(envUrl, 'getCredentials');
+      const response = await getCredentials.fire({ identity });
+
+      assertEquals(response.credentials.length, 1);
+      assertEquals(response.credentials[0].accessKeyId, 'env-key');
+    } finally {
+      if (container) {
+        await container.stop();
+      }
+      if (previousAccessKey === undefined) {
+        Deno.env.delete('OBJST_ACCESS_KEY');
+      } else {
+        Deno.env.set('OBJST_ACCESS_KEY', previousAccessKey);
+      }
+      if (previousSecretKey === undefined) {
+        Deno.env.delete('OBJST_SECRET_KEY');
+      } else {
+        Deno.env.set('OBJST_SECRET_KEY', previousSecretKey);
+      }
+      await cleanupEnvStorageDirectory();
+    }
+  });
+
+  it('does not persist rejected credential replacements', async () => {
+    const portIndex = 11;
+    const rejectPorts = getTestPorts(portIndex);
+    const rejectUrl = `http://localhost:${rejectPorts.uiPort}/typedrequest`;
+    const rejectStorageDirectory = `.nogit/testdata-${portIndex}`;
+    let container: ObjectStorageContainer | null = null;
+
+    const cleanupRejectStorageDirectory = async () => {
+      try {
+        await Deno.remove(rejectStorageDirectory, { recursive: true });
+      } catch (error) {
+        if (!(error instanceof Deno.errors.NotFound)) {
+          throw error;
+        }
+      }
+    };
+
+    try {
+      await cleanupRejectStorageDirectory();
+      container = createTestContainer(portIndex, { storageDirectory: rejectStorageDirectory });
+      await container.start();
+
+      await assertRejects(() =>
+        container!.replaceAccessCredentials([
+          { accessKeyId: 'duplicate-key', secretAccessKey: 'secret-a' },
+          { accessKeyId: 'duplicate-key', secretAccessKey: 'secret-b' },
+        ])
+      );
+
+      const identity = await loginAndGetIdentity(rejectPorts.uiPort);
+      const getCredentials = new TypedRequest<IReq_GetCredentials>(rejectUrl, 'getCredentials');
+      const response = await getCredentials.fire({ identity });
+
+      assertEquals(response.credentials.length, 1);
+      assertEquals(response.credentials[0].accessKeyId, TEST_ACCESS_KEY);
+      await assertRejects(() =>
+        Deno.readTextFile(`${rejectStorageDirectory}/.objectstorage/admin-config.json`)
+      );
+    } finally {
+      if (container) {
+        await container.stop();
+      }
+      await cleanupRejectStorageDirectory();
+    }
+  });
+});

test/test.docker-smoke.test.ts

@@ -0,0 +1,97 @@
+import { assertEquals } from 'jsr:@std/assert';
+
+const shouldRunDockerSmoke = Deno.env.get('OBJST_RUN_DOCKER_SMOKE') === '1';
+
+interface ICommandResult {
+  code: number;
+  stdout: string;
+  stderr: string;
+}
+
+async function runCommand(
+  command: string[],
+  options: { cwd?: string; check?: boolean } = {},
+): Promise<ICommandResult> {
+  const output = await new Deno.Command(command[0], {
+    args: command.slice(1),
+    cwd: options.cwd,
+    stdout: 'piped',
+    stderr: 'piped',
+  }).output();
+
+  const result = {
+    code: output.code,
+    stdout: new TextDecoder().decode(output.stdout).trim(),
+    stderr: new TextDecoder().decode(output.stderr).trim(),
+  };
+
+  if (options.check !== false && result.code !== 0) {
+    throw new Error(`Command failed: ${command.join(' ')}\n${result.stderr}`);
+  }
+
+  return result;
+}
+
+async function waitForOk(url: string, timeoutMs: number): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url);
+      if (response.ok) {
+        return;
+      }
+    } catch {
+      // Container may still be starting.
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+
+  throw new Error(`Timed out waiting for ${url}`);
+}
+
+Deno.test({
+  name: 'Docker image builds and serves the management UI',
+  ignore: !shouldRunDockerSmoke,
+  sanitizeOps: false,
+  sanitizeResources: false,
+  fn: async () => {
+    await runCommand(['docker', '--version']);
+
+    const imageTag = `objectstorage-smoke:${crypto.randomUUID().slice(0, 8)}`;
+    const containerName = `objectstorage-smoke-${crypto.randomUUID().slice(0, 8)}`;
+    const storagePort = 19190;
+    const uiPort = 19191;
+
+    try {
+      await runCommand(['docker', 'build', '-t', imageTag, '.']);
+
+      const runResult = await runCommand([
+        'docker',
+        'run',
+        '-d',
+        '--name',
+        containerName,
+        '-p',
+        `${storagePort}:9000`,
+        '-p',
+        `${uiPort}:3000`,
+        '-e',
+        'OBJST_ADMIN_PASSWORD=docker-smoke-admin',
+        '-e',
+        'OBJST_ACCESS_KEY=docker-smoke-key',
+        '-e',
+        'OBJST_SECRET_KEY=docker-smoke-secret',
+        imageTag,
+      ]);
+
+      assertEquals(runResult.stdout.length > 0, true);
+      await waitForOk(`http://127.0.0.1:${uiPort}/readyz`, 30000);
+      await waitForOk(`http://127.0.0.1:${storagePort}/-/ready`, 30000);
+    } finally {
+      await runCommand(['docker', 'rm', '-f', containerName], { check: false });
+      await runCommand(['docker', 'rmi', '-f', imageTag], { check: false });
+    }
+  },
+});

test/test.enterprise-hardening.test.ts

@@ -0,0 +1,146 @@
+import { assertEquals, assertRejects } from 'jsr:@std/assert';
+import { describe, it } from 'jsr:@std/testing/bdd';
+import { TypedRequest } from '@api.global/typedrequest';
+import {
+  createTestContainer,
+  getTestPorts,
+  loginAndGetIdentity,
+} from './helpers/server.helper.ts';
+import { ObjectStorageContainer } from '../ts/index.ts';
+import type { IReq_AdminLogout, IReq_VerifyIdentity } from '../ts_interfaces/requests/admin.ts';
+import type { IReq_CreateBucket } from '../ts_interfaces/requests/buckets.ts';
+import type { IReq_AddCredential } from '../ts_interfaces/requests/credentials.ts';
+import type { IReq_ListAuditEntries } from '../ts_interfaces/requests/audit.ts';
+
+const PORT_INDEX = 12;
+const ports = getTestPorts(PORT_INDEX);
+const url = `http://localhost:${ports.uiPort}/typedrequest`;
+const storageDirectory = `.nogit/testdata-${PORT_INDEX}`;
+
+const cleanupStorageDirectory = async () => {
+  try {
+    await Deno.remove(storageDirectory, { recursive: true });
+  } catch (error) {
+    if (!(error instanceof Deno.errors.NotFound)) {
+      throw error;
+    }
+  }
+};
+
+describe('Enterprise hardening', { sanitizeResources: false, sanitizeOps: false }, () => {
+  it('refuses default admin credentials on persistent production storage', async () => {
+    const previousAllowInsecureDefaults = Deno.env.get('OBJST_ALLOW_INSECURE_DEFAULTS');
+    const previousAccessKey = Deno.env.get('OBJST_ACCESS_KEY');
+    const previousSecretKey = Deno.env.get('OBJST_SECRET_KEY');
+    const previousAdminPassword = Deno.env.get('OBJST_ADMIN_PASSWORD');
+    const previousStorageDir = Deno.env.get('OBJST_STORAGE_DIR');
+    try {
+      Deno.env.delete('OBJST_ALLOW_INSECURE_DEFAULTS');
+      Deno.env.delete('OBJST_ACCESS_KEY');
+      Deno.env.delete('OBJST_SECRET_KEY');
+      Deno.env.delete('OBJST_ADMIN_PASSWORD');
+      Deno.env.delete('OBJST_STORAGE_DIR');
+      const container = new ObjectStorageContainer();
+      await assertRejects(() => container.start(), Error, 'Refusing to start with default admin credentials');
+    } finally {
+      if (previousAllowInsecureDefaults === undefined) {
+        Deno.env.delete('OBJST_ALLOW_INSECURE_DEFAULTS');
+      } else {
+        Deno.env.set('OBJST_ALLOW_INSECURE_DEFAULTS', previousAllowInsecureDefaults);
+      }
+      if (previousAccessKey === undefined) Deno.env.delete('OBJST_ACCESS_KEY');
+      else Deno.env.set('OBJST_ACCESS_KEY', previousAccessKey);
+      if (previousSecretKey === undefined) Deno.env.delete('OBJST_SECRET_KEY');
+      else Deno.env.set('OBJST_SECRET_KEY', previousSecretKey);
+      if (previousAdminPassword === undefined) Deno.env.delete('OBJST_ADMIN_PASSWORD');
+      else Deno.env.set('OBJST_ADMIN_PASSWORD', previousAdminPassword);
+      if (previousStorageDir === undefined) Deno.env.delete('OBJST_STORAGE_DIR');
+      else Deno.env.set('OBJST_STORAGE_DIR', previousStorageDir);
+    }
+  });
+
+  it('exposes health endpoints and writes audit entries for privileged actions', async () => {
+    await cleanupStorageDirectory();
+    let container: ObjectStorageContainer | null = null;
+
+    try {
+      container = createTestContainer(PORT_INDEX, { storageDirectory });
+      await container.start();
+
+      const live = await fetch(`http://localhost:${ports.uiPort}/livez`);
+      assertEquals(live.status, 200);
+      assertEquals((await live.json()).status, 'alive');
+
+      const ready = await fetch(`http://localhost:${ports.uiPort}/readyz`);
+      assertEquals(ready.status, 200);
+      assertEquals((await ready.json()).status, 'ready');
+
+      const health = await fetch(`http://localhost:${ports.uiPort}/healthz`);
+      assertEquals(health.status, 200);
+      assertEquals((await health.json()).ok, true);
+
+      const metrics = await fetch(`http://localhost:${ports.uiPort}/metrics`);
+      assertEquals(metrics.status, 200);
+      assertEquals((await metrics.text()).includes('objectstorage_ready 1'), true);
+
+      const identity = await loginAndGetIdentity(ports.uiPort);
+      const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
+      await createBucket.fire({ identity, bucketName: 'enterprise-audit-bucket' });
+
+      const addCredential = new TypedRequest<IReq_AddCredential>(url, 'addCredential');
+      await addCredential.fire({
+        identity,
+        accessKeyId: 'enterprise-key',
+        secretAccessKey: 'enterprise-secret',
+      });
+
+      const adminConfigInfo = await Deno.stat(
+        `${storageDirectory}/.objectstorage/admin-config.json`,
+      );
+      if (adminConfigInfo.mode !== null) {
+        assertEquals(adminConfigInfo.mode & 0o777, 0o600);
+      }
+
+      const listAuditEntries = new TypedRequest<IReq_ListAuditEntries>(url, 'listAuditEntries');
+      const auditResponse = await listAuditEntries.fire({ identity, limit: 10 });
+      const actions = auditResponse.entries.map((entry) => entry.action);
+
+      assertEquals(actions.includes('admin.login'), true);
+      assertEquals(actions.includes('bucket.create'), true);
+      assertEquals(actions.includes('credential.add'), true);
+    } finally {
+      if (container) {
+        await container.stop();
+      }
+      await cleanupStorageDirectory();
+    }
+  });
+
+  it('revokes admin identities on logout', async () => {
+    await cleanupStorageDirectory();
+    let container: ObjectStorageContainer | null = null;
+
+    try {
+      container = createTestContainer(PORT_INDEX, { storageDirectory });
+      await container.start();
+
+      const identity = await loginAndGetIdentity(ports.uiPort);
+      const logout = new TypedRequest<IReq_AdminLogout>(url, 'adminLogout');
+      await logout.fire({ identity });
+
+      const verifyIdentity = new TypedRequest<IReq_VerifyIdentity>(url, 'verifyIdentity');
+      const verification = await verifyIdentity.fire({ identity });
+      assertEquals(verification.valid, false);
+
+      const createBucket = new TypedRequest<IReq_CreateBucket>(url, 'createBucket');
+      await assertRejects(() =>
+        createBucket.fire({ identity, bucketName: 'revoked-token-bucket' })
+      );
+    } finally {
+      if (container) {
+        await container.stop();
+      }
+      await cleanupStorageDirectory();
+    }
+  });
+});

test/test.status-config.test.ts

@@ -5,8 +5,11 @@ import { createTestContainer, getTestPorts, loginAndGetIdentity } from './helper
 import { ObjectStorageContainer } from '../ts/index.ts';
 import type * as interfaces from '../ts_interfaces/index.ts';
 import type { IReq_CreateBucket, IReq_DeleteBucket } from '../ts_interfaces/requests/buckets.ts';
-import type { IReq_PutObject, IReq_DeletePrefix } from '../ts_interfaces/requests/objects.ts';
-import type { IReq_GetServerStatus } from '../ts_interfaces/requests/status.ts';
+import type { IReq_DeletePrefix, IReq_PutObject } from '../ts_interfaces/requests/objects.ts';
+import type {
+  IReq_GetClusterHealth,
+  IReq_GetServerStatus,
+} from '../ts_interfaces/requests/status.ts';
 import type { IReq_GetServerConfig } from '../ts_interfaces/requests/config.ts';
 
 const PORT_INDEX = 7;
@@ -94,6 +97,13 @@ describe('Status and config', { sanitizeResources: false, sanitizeOps: false },
     assertEquals(config.corsEnabled, false);
   });
 
+  it('should report standalone cluster health', async () => {
+    const req = new TypedRequest<IReq_GetClusterHealth>(url, 'getClusterHealth');
+    const response = await req.fire({ identity });
+
+    assertEquals(response.clusterHealth.enabled, false);
+  });
+
   it('should reflect correct stats after adding objects', async () => {
     // Add a third object
     const putObj = new TypedRequest<IReq_PutObject>(url, 'putObject');