feat(bucket-tenants): add persisted bucket-scoped tenant credentials with bucket export and import APIs

2026-05-02 11:14:15 +00:00
parent 53d663597a
commit 7f2546e041
14 changed files with 1675 additions and 117 deletions
@@ -0,0 +1,335 @@
+/// <reference types="node" />
+
+import { expect, tap } from '@git.zone/tstest/tapbundle';
+import {
+  CopyObjectCommand,
+  GetBucketPolicyCommand,
+  GetObjectCommand,
+  HeadBucketCommand,
+  ListBucketsCommand,
+  ListObjectsV2Command,
+  PutBucketPolicyCommand,
+  PutObjectCommand,
+  DeleteObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { rm } from 'fs/promises';
+import { fileURLToPath } from 'url';
+import { Readable } from 'stream';
+import * as smartstorage from '../ts/index.js';
+
+const TEST_PORT = 3361;
+const STORAGE_DIR = fileURLToPath(new URL('../.nogit/bucket-tenant-tests', import.meta.url));
+const WORKAPP_A_BUCKET = 'workapp-a-bucket';
+const WORKAPP_B_BUCKET = 'workapp-b-bucket';
+const RESTORE_BUCKET = 'workapp-a-restore-bucket';
+const POLICY_BUCKET = 'workapp-policy-bucket';
+const ADMIN_CREDENTIAL: smartstorage.IStorageCredential = {
+  accessKeyId: 'TENANTADMIN',
+  secretAccessKey: 'TENANTADMINSECRET123',
+};
+
+let testSmartStorageInstance: smartstorage.SmartStorage;
+let adminClient: S3Client;
+let tenantA: smartstorage.IBucketTenantDescriptor;
+let tenantB: smartstorage.IBucketTenantDescriptor;
+let tenantAClient: S3Client;
+let tenantBClient: S3Client;
+let oldTenantAClient: S3Client;
+
+function createS3Client(
+  credential: smartstorage.IStorageCredential,
+  region = 'us-east-1',
+): S3Client {
+  return new S3Client({
+    endpoint: `http://localhost:${TEST_PORT}`,
+    region,
+    credentials: {
+      accessKeyId: credential.accessKeyId,
+      secretAccessKey: credential.secretAccessKey,
+    },
+    forcePathStyle: true,
+  });
+}
+
+function createS3ClientFromDescriptor(
+  descriptor: smartstorage.IBucketTenantDescriptor,
+): S3Client {
+  return new S3Client({
+    endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
+    region: descriptor.region,
+    credentials: {
+      accessKeyId: descriptor.accessKeyId,
+      secretAccessKey: descriptor.secretAccessKey,
+    },
+    forcePathStyle: true,
+  });
+}
+
+async function streamToString(stream: Readable): Promise<string> {
+  const chunks: Buffer[] = [];
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk: string | Buffer | Uint8Array) => chunks.push(Buffer.from(chunk)));
+    stream.on('error', reject);
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+  });
+}
+
+async function startStorage() {
+  testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
+    server: {
+      port: TEST_PORT,
+      silent: true,
+      region: 'us-east-1',
+    },
+    storage: {
+      directory: STORAGE_DIR,
+      cleanSlate: false,
+    },
+    auth: {
+      enabled: true,
+      credentials: [ADMIN_CREDENTIAL],
+    },
+  });
+  adminClient = createS3Client(ADMIN_CREDENTIAL);
+}
+
+tap.test('setup: start storage and provision bucket tenants', async () => {
+  await rm(STORAGE_DIR, { recursive: true, force: true });
+  await startStorage();
+
+  tenantA = await testSmartStorageInstance.createBucketTenant({
+    bucketName: WORKAPP_A_BUCKET,
+  });
+  tenantB = await testSmartStorageInstance.createBucketTenant({
+    bucketName: WORKAPP_B_BUCKET,
+  });
+  tenantAClient = createS3ClientFromDescriptor(tenantA);
+  tenantBClient = createS3ClientFromDescriptor(tenantB);
+});
+
+tap.test('tenant descriptors expose app-ready S3 connection data', async () => {
+  expect(tenantA.endpoint).toEqual('localhost');
+  expect(tenantA.port).toEqual(TEST_PORT);
+  expect(tenantA.region).toEqual('us-east-1');
+  expect(tenantA.bucket).toEqual(WORKAPP_A_BUCKET);
+  expect(tenantA.bucketName).toEqual(WORKAPP_A_BUCKET);
+  expect(tenantA.accessKeyId).toBeTypeofString();
+  expect(tenantA.secretAccessKey).toBeTypeofString();
+  expect(tenantA.useSsl).toEqual(false);
+  expect(tenantA.env.S3_BUCKET).toEqual(WORKAPP_A_BUCKET);
+  expect(tenantA.env.AWS_ACCESS_KEY_ID).toEqual(tenantA.accessKeyId);
+});
+
+tap.test('listBucketTenants returns scoped credential metadata without secrets', async () => {
+  const tenants = await testSmartStorageInstance.listBucketTenants();
+  expect(tenants.length).toEqual(2);
+  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_A_BUCKET)).toEqual(true);
+  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(true);
+  expect((tenants[0] as any).secretAccessKey).toEqual(undefined);
+});
+
+tap.test('tenant credentials work with AWS SDK v3 for their assigned bucket', async () => {
+  const putA = await tenantAClient.send(new PutObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+    Body: 'hello from tenant a',
+    ContentType: 'text/plain',
+  }));
+  expect(putA.$metadata.httpStatusCode).toEqual(200);
+
+  const putB = await tenantBClient.send(new PutObjectCommand({
+    Bucket: WORKAPP_B_BUCKET,
+    Key: 'other.txt',
+    Body: 'hello from tenant b',
+    ContentType: 'text/plain',
+  }));
+  expect(putB.$metadata.httpStatusCode).toEqual(200);
+
+  const getA = await tenantAClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+  }));
+  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');
+
+  const listA = await tenantAClient.send(new ListObjectsV2Command({
+    Bucket: WORKAPP_A_BUCKET,
+  }));
+  expect(listA.Contents?.some((object) => object.Key === 'hello.txt')).toEqual(true);
+});
+
+tap.test('tenant credentials cannot access unrelated buckets', async () => {
+  await expect(tenantAClient.send(new ListBucketsCommand({}))).rejects.toThrow();
+  await expect(tenantAClient.send(new HeadBucketCommand({
+    Bucket: WORKAPP_B_BUCKET,
+  }))).rejects.toThrow();
+  await expect(tenantAClient.send(new PutObjectCommand({
+    Bucket: WORKAPP_B_BUCKET,
+    Key: 'blocked-write.txt',
+    Body: 'blocked',
+  }))).rejects.toThrow();
+  await expect(tenantAClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_B_BUCKET,
+    Key: 'other.txt',
+  }))).rejects.toThrow();
+  await expect(tenantAClient.send(new DeleteObjectCommand({
+    Bucket: WORKAPP_B_BUCKET,
+    Key: 'other.txt',
+  }))).rejects.toThrow();
+  await expect(tenantAClient.send(new CopyObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'copy-from-other-bucket.txt',
+    CopySource: `/${WORKAPP_B_BUCKET}/other.txt`,
+  }))).rejects.toThrow();
+  await expect(tenantBClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+  }))).rejects.toThrow();
+});
+
+tap.test('health and metrics expose running storage state', async () => {
+  const health = await testSmartStorageInstance.getHealth();
+  expect(health.running).toEqual(true);
+  expect(health.ok).toEqual(true);
+  expect(health.storageDirectory).toEqual(STORAGE_DIR);
+  expect(health.auth.enabled).toEqual(true);
+  expect(health.auth.tenantCredentialCount).toEqual(2);
+  expect(health.bucketCount >= 2).toEqual(true);
+  expect(health.objectCount >= 2).toEqual(true);
+  expect(health.totalBytes > 0).toEqual(true);
+
+  const metrics = await testSmartStorageInstance.getMetrics();
+  expect(metrics.tenantCredentialCount).toEqual(2);
+  expect(metrics.prometheusText).toMatch(/smartstorage_tenant_credentials_total 2/);
+});
+
+tap.test('export/import targets one bucket without unrelated tenant data', async () => {
+  const bucketExport = await testSmartStorageInstance.exportBucket({
+    bucketName: WORKAPP_A_BUCKET,
+  });
+  expect(bucketExport.format).toEqual('smartstorage.bucket.v1');
+  expect(bucketExport.bucketName).toEqual(WORKAPP_A_BUCKET);
+  expect(bucketExport.objects.some((object) => object.key === 'hello.txt')).toEqual(true);
+  expect(bucketExport.objects.some((object) => object.key === 'other.txt')).toEqual(false);
+
+  await testSmartStorageInstance.importBucket({
+    bucketName: RESTORE_BUCKET,
+    source: bucketExport,
+  });
+
+  const restoredObject = await adminClient.send(new GetObjectCommand({
+    Bucket: RESTORE_BUCKET,
+    Key: 'hello.txt',
+  }));
+  expect(await streamToString(restoredObject.Body as Readable)).toEqual('hello from tenant a');
+
+  const restoredObjects = await adminClient.send(new ListObjectsV2Command({
+    Bucket: RESTORE_BUCKET,
+  }));
+  expect(restoredObjects.Contents?.some((object) => object.Key === 'other.txt')).toEqual(false);
+});
+
+tap.test('bucket policies persist across restart', async () => {
+  await testSmartStorageInstance.createBucket(POLICY_BUCKET);
+  const policy = JSON.stringify({
+    Version: '2012-10-17',
+    Statement: [{
+      Sid: 'TenantPolicyPersistence',
+      Effect: 'Allow',
+      Principal: { AWS: ADMIN_CREDENTIAL.accessKeyId },
+      Action: ['s3:GetBucketPolicy', 's3:PutBucketPolicy', 's3:ListBucket'],
+      Resource: `arn:aws:s3:::${POLICY_BUCKET}`,
+    }],
+  });
+
+  const response = await adminClient.send(new PutBucketPolicyCommand({
+    Bucket: POLICY_BUCKET,
+    Policy: policy,
+  }));
+  expect(response.$metadata.httpStatusCode).toEqual(204);
+});
+
+tap.test('credential rotation replaces the active tenant credential', async () => {
+  oldTenantAClient = tenantAClient;
+  tenantA = await testSmartStorageInstance.rotateBucketTenantCredentials({
+    bucketName: WORKAPP_A_BUCKET,
+  });
+  tenantAClient = createS3ClientFromDescriptor(tenantA);
+
+  await expect(oldTenantAClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+  }))).rejects.toThrow();
+
+  const getA = await tenantAClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+  }));
+  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');
+
+  const descriptor = await testSmartStorageInstance.getBucketTenantDescriptor({
+    bucketName: WORKAPP_A_BUCKET,
+  });
+  expect(descriptor.accessKeyId).toEqual(tenantA.accessKeyId);
+  expect(descriptor.secretAccessKey).toEqual(tenantA.secretAccessKey);
+});
+
+tap.test('runtime credentials survive restart', async () => {
+  await testSmartStorageInstance.stop();
+  await startStorage();
+
+  const persistedTenantA = await testSmartStorageInstance.getBucketTenantDescriptor({
+    bucketName: WORKAPP_A_BUCKET,
+  });
+  expect(persistedTenantA.accessKeyId).toEqual(tenantA.accessKeyId);
+  expect(persistedTenantA.secretAccessKey).toEqual(tenantA.secretAccessKey);
+
+  tenantAClient = createS3ClientFromDescriptor(persistedTenantA);
+  const getA = await tenantAClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_A_BUCKET,
+    Key: 'hello.txt',
+  }));
+  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');
+
+  const tenants = await testSmartStorageInstance.listBucketTenants();
+  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_A_BUCKET)).toEqual(true);
+  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(true);
+
+  const policyResponse = await adminClient.send(new GetBucketPolicyCommand({
+    Bucket: POLICY_BUCKET,
+  }));
+  expect(policyResponse.Policy?.includes('TenantPolicyPersistence')).toEqual(true);
+});
+
+tap.test('deleteBucketTenant can revoke credentials and delete tenant buckets', async () => {
+  await testSmartStorageInstance.deleteBucketTenant({
+    bucketName: WORKAPP_B_BUCKET,
+    accessKeyId: tenantB.accessKeyId,
+  });
+
+  await expect(tenantBClient.send(new GetObjectCommand({
+    Bucket: WORKAPP_B_BUCKET,
+    Key: 'other.txt',
+  }))).rejects.toThrow();
+
+  const headAfterRevoke = await adminClient.send(new HeadBucketCommand({
+    Bucket: WORKAPP_B_BUCKET,
+  }));
+  expect(headAfterRevoke.$metadata.httpStatusCode).toEqual(200);
+
+  await testSmartStorageInstance.deleteBucketTenant({
+    bucketName: WORKAPP_B_BUCKET,
+  });
+  await expect(adminClient.send(new HeadBucketCommand({
+    Bucket: WORKAPP_B_BUCKET,
+  }))).rejects.toThrow();
+
+  const tenants = await testSmartStorageInstance.listBucketTenants();
+  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(false);
+});
+
+tap.test('teardown: stop storage server', async () => {
+  await testSmartStorageInstance.stop();
+});
+
+export default tap.start();