feat: enhance storage stats and cluster health reporting
- Introduced new data structures for bucket and storage statistics, including BucketSummary, StorageStats, and ClusterHealth. - Implemented runtime statistics tracking for buckets, including object count and total size. - Added methods to retrieve storage stats and bucket summaries in the FileStore. - Enhanced the SmartStorage interface to expose storage stats and cluster health. - Implemented tests for runtime stats, cluster health, and credential management. - Added support for runtime-managed credentials with atomic replacement. - Improved filesystem usage reporting for storage locations.
This commit is contained in:
+112
-6
@@ -1,16 +1,28 @@
|
||||
/// <reference types="node" />
|
||||
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
|
||||
import { Buffer } from 'buffer';
|
||||
import { Readable } from 'stream';
|
||||
import * as smartstorage from '../ts/index.js';
|
||||
|
||||
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||
let s3Client: S3Client;
|
||||
const testObjectBody = 'Hello from AWS SDK!';
|
||||
const testObjectSize = Buffer.byteLength(testObjectBody);
|
||||
|
||||
function getBucketSummary(
|
||||
summaries: smartstorage.IBucketSummary[],
|
||||
bucketName: string,
|
||||
): smartstorage.IBucketSummary | undefined {
|
||||
return summaries.find((summary) => summary.name === bucketName);
|
||||
}
|
||||
|
||||
// Helper to convert stream to string
|
||||
async function streamToString(stream: Readable): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
||||
stream.on('data', (chunk: string | Buffer | Uint8Array) => chunks.push(Buffer.from(chunk)));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
});
|
||||
@@ -46,28 +58,82 @@ tap.test('should list buckets (empty)', async () => {
|
||||
expect(response.Buckets!.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should expose empty runtime stats after startup', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
expect(stats.bucketCount).toEqual(0);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(stats.buckets.length).toEqual(0);
|
||||
expect(stats.storageDirectory.length > 0).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should expose disabled cluster health in standalone mode', async () => {
|
||||
const clusterHealth = await testSmartStorageInstance.getClusterHealth();
|
||||
expect(clusterHealth.enabled).toEqual(false);
|
||||
expect(clusterHealth.nodeId).toEqual(undefined);
|
||||
expect(clusterHealth.quorumHealthy).toEqual(undefined);
|
||||
expect(clusterHealth.drives).toEqual(undefined);
|
||||
});
|
||||
|
||||
tap.test('should create a bucket', async () => {
|
||||
const response = await s3Client.send(new CreateBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('should list buckets (showing created bucket)', async () => {
|
||||
tap.test('should create an empty bucket through the bridge', async () => {
|
||||
const response = await testSmartStorageInstance.createBucket('empty-bucket');
|
||||
expect(response.name).toEqual('empty-bucket');
|
||||
});
|
||||
|
||||
tap.test('should list buckets (showing created buckets)', async () => {
|
||||
const response = await s3Client.send(new ListBucketsCommand({}));
|
||||
expect(response.Buckets!.length).toEqual(1);
|
||||
expect(response.Buckets![0].Name).toEqual('test-bucket');
|
||||
expect(response.Buckets!.length).toEqual(2);
|
||||
expect(response.Buckets!.some((bucket) => bucket.Name === 'test-bucket')).toEqual(true);
|
||||
expect(response.Buckets!.some((bucket) => bucket.Name === 'empty-bucket')).toEqual(true);
|
||||
});
|
||||
|
||||
tap.test('should expose runtime bucket summaries after bucket creation', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const summaries = await testSmartStorageInstance.listBucketSummaries();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(summaries, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(summaries.length).toEqual(2);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(typeof testBucketSummary?.creationDate).toEqual('number');
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should upload an object', async () => {
|
||||
const response = await s3Client.send(new PutObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
Body: 'Hello from AWS SDK!',
|
||||
Body: testObjectBody,
|
||||
ContentType: 'text/plain',
|
||||
}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
expect(response.ETag).toBeTypeofString();
|
||||
});
|
||||
|
||||
tap.test('should reflect uploaded object in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(1);
|
||||
expect(stats.totalStorageBytes).toEqual(testObjectSize);
|
||||
expect(testBucketSummary?.objectCount).toEqual(1);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(testObjectSize);
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should download the object', async () => {
|
||||
const response = await s3Client.send(new GetObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
@@ -76,7 +142,7 @@ tap.test('should download the object', async () => {
|
||||
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
const content = await streamToString(response.Body as Readable);
|
||||
expect(content).toEqual('Hello from AWS SDK!');
|
||||
expect(content).toEqual(testObjectBody);
|
||||
});
|
||||
|
||||
tap.test('should delete the object', async () => {
|
||||
@@ -87,6 +153,20 @@ tap.test('should delete the object', async () => {
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should reflect object deletion in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(2);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(emptyBucketSummary?.objectCount).toEqual(0);
|
||||
expect(emptyBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should fail to get deleted object', async () => {
|
||||
await expect(
|
||||
s3Client.send(new GetObjectCommand({
|
||||
@@ -96,11 +176,37 @@ tap.test('should fail to get deleted object', async () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
tap.test('should delete the empty bucket', async () => {
|
||||
const response = await s3Client.send(new DeleteBucketCommand({ Bucket: 'empty-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should reflect bucket deletion in runtime stats', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
const testBucketSummary = getBucketSummary(stats.buckets, 'test-bucket');
|
||||
const emptyBucketSummary = getBucketSummary(stats.buckets, 'empty-bucket');
|
||||
|
||||
expect(stats.bucketCount).toEqual(1);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(testBucketSummary?.objectCount).toEqual(0);
|
||||
expect(testBucketSummary?.totalSizeBytes).toEqual(0);
|
||||
expect(emptyBucketSummary).toEqual(undefined);
|
||||
});
|
||||
|
||||
tap.test('should delete the bucket', async () => {
|
||||
const response = await s3Client.send(new DeleteBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should expose empty runtime stats after deleting all buckets', async () => {
|
||||
const stats = await testSmartStorageInstance.getStorageStats();
|
||||
expect(stats.bucketCount).toEqual(0);
|
||||
expect(stats.totalObjectCount).toEqual(0);
|
||||
expect(stats.totalStorageBytes).toEqual(0);
|
||||
expect(stats.buckets.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should stop the storage server', async () => {
|
||||
await testSmartStorageInstance.stop();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user