feat: enhance storage stats and cluster health reporting
- Introduced new data structures for bucket and storage statistics, including BucketSummary, StorageStats, and ClusterHealth. - Implemented runtime statistics tracking for buckets, including object count and total size. - Added methods to retrieve storage stats and bucket summaries in the FileStore. - Enhanced the SmartStorage interface to expose storage stats and cluster health. - Implemented tests for runtime stats, cluster health, and credential management. - Added support for runtime-managed credentials with atomic replacement. - Improved filesystem usage reporting for storage locations.
This commit is contained in:
+112
-6
@@ -1,16 +1,28 @@
|
||||
/// <reference types="node" />
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
|
||||
import { Buffer } from 'buffer';
|
||||
import { Readable } from 'stream';
|
||||
import * as smartstorage from '../ts/index.js';
|
||||
|
||||
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||
let s3Client: S3Client;
|
||||
const testObjectBody = 'Hello from AWS SDK!';
|
||||
const testObjectSize = Buffer.byteLength(testObjectBody);
|
||||
|
||||
function getBucketSummary(
|
||||
summaries: smartstorage.IBucketSummary[],
|
||||
bucketName: string,
|
||||
): smartstorage.IBucketSummary | undefined {
|
||||
return summaries.find((summary) => summary.name === bucketName);
|
||||
}
|
||||
|
||||
// Helper to convert stream to string
|
||||
async function streamToString(stream: Readable): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
||||
stream.on('data', (chunk: string | Buffer | Uint8Array) => chunks.push(Buffer.from(chunk)));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
});
|
||||
@@ -46,28 +58,82 @@ tap.test('should list buckets (empty)', async () => {
|
||||
expect(response.Buckets!.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should expose empty runtime stats after startup', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
expect(stats.bucketCount).toEqual(0);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(stats.buckets.length).toEqual(0);
|
||||
expect(stats.storageDirectory.length > 0).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should expose disabled cluster health in standalone mode', async () => {
|
||||
const clusterHealth = await testSmartStorageInstance.getClusterHealth();
|
||||
expect(clusterHealth.enabled).toEqual(false);
|
||||
expect(clusterHealth.nodeId).toEqual(undefined);
|
||||
expect(clusterHealth.quorumHealthy).toEqual(undefined);
|
||||
expect(clusterHealth.drives).toEqual(undefined);
|
||||
});
|
||||
|
||||
tap.test('should create a bucket', async () => {
|
||||
const response = await s3Client.send(new CreateBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('should list buckets (showing created bucket)', async () => {
|
||||
tap.test('should create an empty bucket through the bridge', async () => {
|
||||
const response = await testSmartStorageInstance.createBucket('empty-bucket');
|
||||
expect(response.name).toEqual('empty-bucket');
|
||||
});
|
||||
|
||||
tap.test('should list buckets (showing created buckets)', async () => {
|
||||
const response = await s3Client.send(new ListBucketsCommand({}));
|
||||
expect(response.Buckets!.length).toEqual(1);
|
||||
expect(response.Buckets![0].Name).toEqual('test-bucket');
|
||||
expect(response.Buckets!.length).toEqual(2);
|
||||
expect(response.Buckets!.some((bucket) => bucket.Name === 'test-bucket')).toEqual(true);
|
||||
expect(response.Buckets!.some((bucket) => bucket.Name === 'empty-bucket')).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should expose runtime bucket summaries after bucket creation', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const summaries = await testSmartStorageInstance.listBucketSummaries();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(summaries, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(summaries.length).toEqual(2);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(typeof testBucketSummary?.creationDate).toEqual('number');
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should upload an object', async () => {
|
||||
const response = await s3Client.send(new PutObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
Body: 'Hello from AWS SDK!',
|
||||
Body: testObjectBody,
|
||||
ContentType: 'text/plain',
|
||||
}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
expect(response.ETag).toBeTypeofString();
|
||||
});
|
||||
|
||||
tap.test('should reflect uploaded object in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(1);
|
||||
expect(stats.totalStorageBytes).toEqual(testObjectSize);
|
||||
expect(testBucketSummary?.objectCount).toEqual(1);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(testObjectSize);
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should download the object', async () => {
|
||||
const response = await s3Client.send(new GetObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
@@ -76,7 +142,7 @@ tap.test('should download the object', async () => {
|
||||
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
const content = await streamToString(response.Body as Readable);
|
||||
expect(content).toEqual('Hello from AWS SDK!');
|
||||
expect(content).toEqual(testObjectBody);
|
||||
});
|
||||
|
||||
tap.test('should delete the object', async () => {
|
||||
@@ -87,6 +153,20 @@ tap.test('should delete the object', async () => {
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should reflect object deletion in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should fail to get deleted object', async () => {
|
||||
await expect(
|
||||
s3Client.send(new GetObjectCommand({
|
||||
@@ -96,11 +176,37 @@ tap.test('should fail to get deleted object', async () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
tap.test('should delete the empty bucket', async () => {
|
||||
const response = await s3Client.send(new DeleteBucketCommand({ Bucket: 'empty-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should reflect bucket deletion in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(1);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(emptyBucketSummary).toEqual(undefined);
|
||||
});
|
||||
|
||||
tap.test('should delete the bucket', async () => {
|
||||
const response = await s3Client.send(new DeleteBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should expose empty runtime stats after deleting all buckets', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
expect(stats.bucketCount).toEqual(0);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(stats.buckets.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should stop the storage server', async () => {
|
||||
await testSmartStorageInstance.stop();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
/// <reference types="node" />
|
||||
|
||||
import { rm } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as smartstorage from '../ts/index.js';
|
||||
|
||||
let clusterStorage: smartstorage.SmartStorage;
|
||||
const baseDir = join(process.cwd(), '.nogit', `cluster-health-${Date.now()}`);
|
||||
const drivePaths = Array.from({ length: 6 }, (_value, index) => {
|
||||
return join(baseDir, `drive-${index + 1}`);
|
||||
});
|
||||
const storageDir = join(baseDir, 'storage');
|
||||
|
||||
tap.test('setup: start clustered storage server', async () => {
|
||||
clusterStorage = await smartstorage.SmartStorage.createAndStart({
|
||||
server: {
|
||||
port: 3348,
|
||||
silent: true,
|
||||
},
|
||||
storage: {
|
||||
directory: storageDir,
|
||||
},
|
||||
cluster: {
|
||||
enabled: true,
|
||||
nodeId: 'cluster-health-node',
|
||||
quicPort: 4348,
|
||||
seedNodes: [],
|
||||
erasure: {
|
||||
dataShards: 4,
|
||||
parityShards: 2,
|
||||
chunkSizeBytes: 1024 * 1024,
|
||||
},
|
||||
drives: {
|
||||
paths: drivePaths,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('should expose clustered runtime health', async () => {
|
||||
const health = await clusterStorage.getClusterHealth();
|
||||
|
||||
expect(health.enabled).toEqual(true);
|
||||
expect(health.nodeId).toEqual('cluster-health-node');
|
||||
expect(health.quorumHealthy).toEqual(true);
|
||||
expect(health.majorityHealthy).toEqual(true);
|
||||
expect(Array.isArray(health.peers)).toEqual(true);
|
||||
expect(health.peers!.length).toEqual(0);
|
||||
expect(Array.isArray(health.drives)).toEqual(true);
|
||||
expect(health.drives!.length).toEqual(6);
|
||||
expect(health.drives!.every((drive) => drive.status === 'online')).toEqual(true);
|
||||
expect(health.drives!.every((drive) => drivePaths.includes(drive.path))).toEqual(true);
|
||||
expect(health.drives!.every((drive) => drive.totalBytes !== undefined)).toEqual(true);
|
||||
expect(health.drives!.every((drive) => drive.usedBytes !== undefined)).toEqual(true);
|
||||
expect(health.drives!.every((drive) => drive.lastCheck !== undefined)).toEqual(true);
|
||||
expect(health.drives!.every((drive) => drive.erasureSetId === 0)).toEqual(true);
|
||||
expect(health.erasure?.dataShards).toEqual(4);
|
||||
expect(health.erasure?.parityShards).toEqual(2);
|
||||
expect(health.erasure?.chunkSizeBytes).toEqual(1024 * 1024);
|
||||
expect(health.erasure?.totalShards).toEqual(6);
|
||||
expect(health.erasure?.readQuorum).toEqual(4);
|
||||
expect(health.erasure?.writeQuorum).toEqual(5);
|
||||
expect(health.erasure?.erasureSetCount).toEqual(1);
|
||||
expect(health.repairs?.active).toEqual(false);
|
||||
expect(health.repairs?.scanIntervalMs).toEqual(24 * 60 * 60 * 1000);
|
||||
});
|
||||
|
||||
tap.test('should expose cluster health after bucket creation', async () => {
|
||||
const bucket = await clusterStorage.createBucket('cluster-health-bucket');
|
||||
const health = await clusterStorage.getClusterHealth();
|
||||
|
||||
expect(bucket.name).toEqual('cluster-health-bucket');
|
||||
expect(health.enabled).toEqual(true);
|
||||
expect(health.quorumHealthy).toEqual(true);
|
||||
expect(health.drives!.length).toEqual(6);
|
||||
});
|
||||
|
||||
tap.test('teardown: stop clustered server and clean files', async () => {
|
||||
await clusterStorage.stop();
|
||||
await rm(baseDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
export default tap.start()
|
||||
@@ -0,0 +1,150 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import {
|
||||
CreateBucketCommand,
|
||||
DeleteBucketCommand,
|
||||
ListBucketsCommand,
|
||||
S3Client,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import * as smartstorage from '../ts/index.js';
|
||||
|
||||
const TEST_PORT = 3349;
|
||||
const INITIAL_CREDENTIAL: smartstorage.IStorageCredential = {
|
||||
accessKeyId: 'RUNTIMEINITIAL',
|
||||
secretAccessKey: 'RUNTIMEINITIALSECRET123',
|
||||
};
|
||||
const ROTATED_CREDENTIAL_A: smartstorage.IStorageCredential = {
|
||||
accessKeyId: 'RUNTIMEA',
|
||||
secretAccessKey: 'RUNTIMEASECRET123',
|
||||
};
|
||||
const ROTATED_CREDENTIAL_B: smartstorage.IStorageCredential = {
|
||||
accessKeyId: 'RUNTIMEB',
|
||||
secretAccessKey: 'RUNTIMEBSECRET123',
|
||||
};
|
||||
const TEST_BUCKET = 'runtime-credentials-bucket';
|
||||
|
||||
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||
let initialClient: S3Client;
|
||||
let rotatedClientA: S3Client;
|
||||
let rotatedClientB: S3Client;
|
||||
|
||||
function createS3Client(credential: smartstorage.IStorageCredential): S3Client {
|
||||
return new S3Client({
|
||||
endpoint: `http://localhost:${TEST_PORT}`,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: credential.accessKeyId,
|
||||
secretAccessKey: credential.secretAccessKey,
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
}
|
||||
|
||||
tap.test('setup: start storage server with runtime-managed credentials', async () => {
|
||||
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||
server: {
|
||||
port: TEST_PORT,
|
||||
silent: true,
|
||||
region: 'us-east-1',
|
||||
},
|
||||
storage: {
|
||||
cleanSlate: true,
|
||||
},
|
||||
auth: {
|
||||
enabled: true,
|
||||
credentials: [INITIAL_CREDENTIAL],
|
||||
},
|
||||
});
|
||||
|
||||
initialClient = createS3Client(INITIAL_CREDENTIAL);
|
||||
rotatedClientA = createS3Client(ROTATED_CREDENTIAL_A);
|
||||
rotatedClientB = createS3Client(ROTATED_CREDENTIAL_B);
|
||||
});
|
||||
|
||||
tap.test('startup credentials authenticate successfully', async () => {
|
||||
const response = await initialClient.send(new ListBucketsCommand({}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('listCredentials returns the active startup credential set', async () => {
|
||||
const credentials = await testSmartStorageInstance.listCredentials();
|
||||
expect(credentials.length).toEqual(1);
|
||||
expect(credentials[0].accessKeyId).toEqual(INITIAL_CREDENTIAL.accessKeyId);
|
||||
expect(credentials[0].secretAccessKey).toEqual(INITIAL_CREDENTIAL.secretAccessKey);
|
||||
});
|
||||
|
||||
tap.test('invalid replacement input fails cleanly and leaves old credentials active', async () => {
|
||||
await expect(
|
||||
testSmartStorageInstance.replaceCredentials([
|
||||
{
|
||||
accessKeyId: '',
|
||||
secretAccessKey: 'invalid-secret',
|
||||
},
|
||||
]),
|
||||
).rejects.toThrow();
|
||||
|
||||
const credentials = await testSmartStorageInstance.listCredentials();
|
||||
expect(credentials.length).toEqual(1);
|
||||
expect(credentials[0].accessKeyId).toEqual(INITIAL_CREDENTIAL.accessKeyId);
|
||||
|
||||
const response = await initialClient.send(new ListBucketsCommand({}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('replacing credentials swaps the active set atomically', async () => {
|
||||
await testSmartStorageInstance.replaceCredentials([
|
||||
ROTATED_CREDENTIAL_A,
|
||||
ROTATED_CREDENTIAL_B,
|
||||
]);
|
||||
|
||||
const credentials = await testSmartStorageInstance.listCredentials();
|
||||
expect(credentials.length).toEqual(2);
|
||||
expect(credentials[0].accessKeyId).toEqual(ROTATED_CREDENTIAL_A.accessKeyId);
|
||||
expect(credentials[1].accessKeyId).toEqual(ROTATED_CREDENTIAL_B.accessKeyId);
|
||||
});
|
||||
|
||||
tap.test('old credentials stop working immediately for new requests', async () => {
|
||||
await expect(initialClient.send(new ListBucketsCommand({}))).rejects.toThrow();
|
||||
});
|
||||
|
||||
tap.test('first rotated credential authenticates successfully', async () => {
|
||||
const response = await rotatedClientA.send(
|
||||
new CreateBucketCommand({ Bucket: TEST_BUCKET }),
|
||||
);
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('multiple rotated credentials remain active', async () => {
|
||||
const response = await rotatedClientB.send(new ListBucketsCommand({}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
expect(response.Buckets?.some((bucket) => bucket.Name === TEST_BUCKET)).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('duplicate replacement input fails cleanly without changing the active set', async () => {
|
||||
await expect(
|
||||
testSmartStorageInstance.replaceCredentials([
|
||||
ROTATED_CREDENTIAL_A,
|
||||
{
|
||||
accessKeyId: ROTATED_CREDENTIAL_A.accessKeyId,
|
||||
secretAccessKey: 'another-secret',
|
||||
},
|
||||
]),
|
||||
).rejects.toThrow();
|
||||
|
||||
const credentials = await testSmartStorageInstance.listCredentials();
|
||||
expect(credentials.length).toEqual(2);
|
||||
expect(credentials[0].accessKeyId).toEqual(ROTATED_CREDENTIAL_A.accessKeyId);
|
||||
expect(credentials[1].accessKeyId).toEqual(ROTATED_CREDENTIAL_B.accessKeyId);
|
||||
|
||||
const response = await rotatedClientA.send(new ListBucketsCommand({}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('teardown: clean up bucket and stop the storage server', async () => {
|
||||
const response = await rotatedClientA.send(
|
||||
new DeleteBucketCommand({ Bucket: TEST_BUCKET }),
|
||||
);
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
await testSmartStorageInstance.stop();
|
||||
});
|
||||
|
||||
export default tap.start()
|
||||
Reference in New Issue
Block a user