smartstorage/test/test.bucket-tenants.node.ts

/// <reference types="node" />

import { expect, tap } from '@git.zone/tstest/tapbundle';
import {
  CopyObjectCommand,
  GetBucketPolicyCommand,
  GetObjectCommand,
  HeadBucketCommand,
  ListBucketsCommand,
  ListObjectsV2Command,
  PutBucketPolicyCommand,
  PutObjectCommand,
  DeleteObjectCommand,
  S3Client,
} from '@aws-sdk/client-s3';
import { rm } from 'fs/promises';
import { fileURLToPath } from 'url';
import { Readable } from 'stream';
import * as smartstorage from '../ts/index.js';

const TEST_PORT = 3361;
const STORAGE_DIR = fileURLToPath(new URL('../.nogit/bucket-tenant-tests', import.meta.url));
const WORKAPP_A_BUCKET = 'workapp-a-bucket';
const WORKAPP_B_BUCKET = 'workapp-b-bucket';
const RESTORE_BUCKET = 'workapp-a-restore-bucket';
const CORRUPT_RESTORE_BUCKET = 'workapp-a-corrupt-restore-bucket';
const POLICY_BUCKET = 'workapp-policy-bucket';
const DUPLICATE_TENANT_BUCKET = 'workapp-duplicate-tenant-bucket';
const REVOKE_ONLY_BUCKET = 'workapp-revoke-only-bucket';
const ADMIN_CREDENTIAL: smartstorage.IStorageCredential = {
  accessKeyId: 'TENANTADMIN',
  secretAccessKey: 'TENANTADMINSECRET123',
};

let testSmartStorageInstance: smartstorage.SmartStorage;
let adminClient: S3Client;
let tenantA: smartstorage.IBucketTenantDescriptor;
let tenantB: smartstorage.IBucketTenantDescriptor;
let tenantAClient: S3Client;
let tenantBClient: S3Client;
let oldTenantAClient: S3Client;

function createS3Client(
  credential: smartstorage.IStorageCredential,
  region = 'us-east-1',
): S3Client {
  return new S3Client({
    endpoint: `http://localhost:${TEST_PORT}`,
    region,
    credentials: {
      accessKeyId: credential.accessKeyId,
      secretAccessKey: credential.secretAccessKey,
    },
    forcePathStyle: true,
  });
}

function createS3ClientFromDescriptor(
  descriptor: smartstorage.IBucketTenantDescriptor,
): S3Client {
  return new S3Client({
    endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
    region: descriptor.region,
    credentials: {
      accessKeyId: descriptor.accessKeyId,
      secretAccessKey: descriptor.secretAccessKey,
    },
    forcePathStyle: true,
  });
}

async function streamToString(stream: Readable): Promise<string> {
  const chunks: Buffer[] = [];
  return new Promise((resolve, reject) => {
    stream.on('data', (chunk: string | Buffer | Uint8Array) => chunks.push(Buffer.from(chunk)));
    stream.on('error', reject);
    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
  });
}

async function startStorage() {
  testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
    server: {
      port: TEST_PORT,
      silent: true,
      region: 'us-east-1',
    },
    storage: {
      directory: STORAGE_DIR,
      cleanSlate: false,
    },
    auth: {
      enabled: true,
      credentials: [ADMIN_CREDENTIAL],
    },
  });
  adminClient = createS3Client(ADMIN_CREDENTIAL);
}

tap.test('bucket tenant client APIs require auth before IPC', async () => {
  const storage = new smartstorage.SmartStorage({
    auth: {
      enabled: false,
      credentials: [ADMIN_CREDENTIAL],
    },
  });

  await expect(storage.listBucketTenants()).rejects.toThrow();
  await expect(storage.getBucketTenantDescriptor({
    bucketName: WORKAPP_A_BUCKET,
  })).rejects.toThrow();
});

tap.test('setup: start storage and provision bucket tenants', async () => {
  await rm(STORAGE_DIR, { recursive: true, force: true });
  await startStorage();

  tenantA = await testSmartStorageInstance.createBucketTenant({
    bucketName: WORKAPP_A_BUCKET,
  });
  tenantB = await testSmartStorageInstance.createBucketTenant({
    bucketName: WORKAPP_B_BUCKET,
  });
  tenantAClient = createS3ClientFromDescriptor(tenantA);
  tenantBClient = createS3ClientFromDescriptor(tenantB);
});

tap.test('tenant descriptors expose app-ready S3 connection data', async () => {
  expect(tenantA.endpoint).toEqual('localhost');
  expect(tenantA.port).toEqual(TEST_PORT);
  expect(tenantA.region).toEqual('us-east-1');
  expect(tenantA.bucket).toEqual(WORKAPP_A_BUCKET);
  expect(tenantA.bucketName).toEqual(WORKAPP_A_BUCKET);
  expect(tenantA.accessKeyId).toBeTypeofString();
  expect(tenantA.secretAccessKey).toBeTypeofString();
  expect(tenantA.useSsl).toEqual(false);
  expect(tenantA.env.S3_BUCKET).toEqual(WORKAPP_A_BUCKET);
  expect(tenantA.env.AWS_ACCESS_KEY_ID).toEqual(tenantA.accessKeyId);
});

tap.test('listBucketTenants returns scoped credential metadata without secrets', async () => {
  const tenants = await testSmartStorageInstance.listBucketTenants();
  expect(tenants.length).toEqual(2);
  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_A_BUCKET)).toEqual(true);
  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(true);
  expect((tenants[0] as any).secretAccessKey).toEqual(undefined);
});

tap.test('createBucketTenant validates credentials before creating buckets', async () => {
  await expect(testSmartStorageInstance.createBucketTenant({
    bucketName: DUPLICATE_TENANT_BUCKET,
    accessKeyId: ADMIN_CREDENTIAL.accessKeyId,
    secretAccessKey: 'DUPLICATESECRET123',
  })).rejects.toThrow();

  await expect(adminClient.send(new HeadBucketCommand({
    Bucket: DUPLICATE_TENANT_BUCKET,
  }))).rejects.toThrow();
});

tap.test('tenant credentials work with AWS SDK v3 for their assigned bucket', async () => {
  const putA = await tenantAClient.send(new PutObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
    Body: 'hello from tenant a',
    ContentType: 'text/plain',
  }));
  expect(putA.$metadata.httpStatusCode).toEqual(200);

  const putB = await tenantBClient.send(new PutObjectCommand({
    Bucket: WORKAPP_B_BUCKET,
    Key: 'other.txt',
    Body: 'hello from tenant b',
    ContentType: 'text/plain',
  }));
  expect(putB.$metadata.httpStatusCode).toEqual(200);

  const getA = await tenantAClient.send(new GetObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
  }));
  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');

  const listA = await tenantAClient.send(new ListObjectsV2Command({
    Bucket: WORKAPP_A_BUCKET,
  }));
  expect(listA.Contents?.some((object) => object.Key === 'hello.txt')).toEqual(true);
});

tap.test('tenant credentials cannot access unrelated buckets', async () => {
  await expect(tenantAClient.send(new ListBucketsCommand({}))).rejects.toThrow();
  await expect(tenantAClient.send(new HeadBucketCommand({
    Bucket: WORKAPP_B_BUCKET,
  }))).rejects.toThrow();
  await expect(tenantAClient.send(new PutObjectCommand({
    Bucket: WORKAPP_B_BUCKET,
    Key: 'blocked-write.txt',
    Body: 'blocked',
  }))).rejects.toThrow();
  await expect(tenantAClient.send(new GetObjectCommand({
    Bucket: WORKAPP_B_BUCKET,
    Key: 'other.txt',
  }))).rejects.toThrow();
  await expect(tenantAClient.send(new DeleteObjectCommand({
    Bucket: WORKAPP_B_BUCKET,
    Key: 'other.txt',
  }))).rejects.toThrow();
  await expect(tenantAClient.send(new CopyObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'copy-from-other-bucket.txt',
    CopySource: `/${WORKAPP_B_BUCKET}/other.txt`,
  }))).rejects.toThrow();
  await expect(tenantBClient.send(new GetObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
  }))).rejects.toThrow();
});

tap.test('health and metrics expose running storage state', async () => {
  const health = await testSmartStorageInstance.getHealth();
  expect(health.running).toEqual(true);
  expect(health.ok).toEqual(true);
  expect(health.storageDirectory).toEqual(STORAGE_DIR);
  expect(health.auth.enabled).toEqual(true);
  expect(health.auth.tenantCredentialCount).toEqual(2);
  expect(health.bucketCount >= 2).toEqual(true);
  expect(health.objectCount >= 2).toEqual(true);
  expect(health.totalBytes > 0).toEqual(true);

  const metrics = await testSmartStorageInstance.getMetrics();
  expect(metrics.tenantCredentialCount).toEqual(2);
  expect(metrics.prometheusText).toMatch(/smartstorage_tenant_credentials_total 2/);
});

tap.test('export/import targets one bucket without unrelated tenant data', async () => {
  const bucketExport = await testSmartStorageInstance.exportBucket({
    bucketName: WORKAPP_A_BUCKET,
  });
  expect(bucketExport.format).toEqual('smartstorage.bucket.v1');
  expect(bucketExport.bucketName).toEqual(WORKAPP_A_BUCKET);
  expect(bucketExport.objects.some((object) => object.key === 'hello.txt')).toEqual(true);
  expect(bucketExport.objects.some((object) => object.key === 'other.txt')).toEqual(false);

  await testSmartStorageInstance.importBucket({
    bucketName: RESTORE_BUCKET,
    source: bucketExport,
  });

  const restoredObject = await adminClient.send(new GetObjectCommand({
    Bucket: RESTORE_BUCKET,
    Key: 'hello.txt',
  }));
  expect(await streamToString(restoredObject.Body as Readable)).toEqual('hello from tenant a');

  const restoredObjects = await adminClient.send(new ListObjectsV2Command({
    Bucket: RESTORE_BUCKET,
  }));
  expect(restoredObjects.Contents?.some((object) => object.Key === 'other.txt')).toEqual(false);

  const corruptExport: smartstorage.IBucketExport = {
    ...bucketExport,
    objects: bucketExport.objects.map((object, index) => index === 0 ? {
      ...object,
      size: object.size + 1,
    } : object),
  };
  await expect(testSmartStorageInstance.importBucket({
    bucketName: CORRUPT_RESTORE_BUCKET,
    source: corruptExport,
  })).rejects.toThrow();
  await expect(adminClient.send(new HeadBucketCommand({
    Bucket: CORRUPT_RESTORE_BUCKET,
  }))).rejects.toThrow();
});

tap.test('bucket policies persist across restart', async () => {
  await testSmartStorageInstance.createBucket(POLICY_BUCKET);
  const policy = JSON.stringify({
    Version: '2012-10-17',
    Statement: [{
      Sid: 'TenantPolicyPersistence',
      Effect: 'Allow',
      Principal: { AWS: ADMIN_CREDENTIAL.accessKeyId },
      Action: ['s3:GetBucketPolicy', 's3:PutBucketPolicy', 's3:ListBucket'],
      Resource: `arn:aws:s3:::${POLICY_BUCKET}`,
    }],
  });

  const response = await adminClient.send(new PutBucketPolicyCommand({
    Bucket: POLICY_BUCKET,
    Policy: policy,
  }));
  expect(response.$metadata.httpStatusCode).toEqual(204);
});

tap.test('credential rotation replaces the active tenant credential', async () => {
  oldTenantAClient = tenantAClient;
  tenantA = await testSmartStorageInstance.rotateBucketTenantCredentials({
    bucketName: WORKAPP_A_BUCKET,
  });
  tenantAClient = createS3ClientFromDescriptor(tenantA);

  await expect(oldTenantAClient.send(new GetObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
  }))).rejects.toThrow();

  const getA = await tenantAClient.send(new GetObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
  }));
  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');

  const descriptor = await testSmartStorageInstance.getBucketTenantDescriptor({
    bucketName: WORKAPP_A_BUCKET,
  });
  expect(descriptor.accessKeyId).toEqual(tenantA.accessKeyId);
  expect(descriptor.secretAccessKey).toEqual(tenantA.secretAccessKey);
});

tap.test('runtime credentials survive restart', async () => {
  await testSmartStorageInstance.stop();
  await startStorage();

  const persistedTenantA = await testSmartStorageInstance.getBucketTenantDescriptor({
    bucketName: WORKAPP_A_BUCKET,
  });
  expect(persistedTenantA.accessKeyId).toEqual(tenantA.accessKeyId);
  expect(persistedTenantA.secretAccessKey).toEqual(tenantA.secretAccessKey);

  tenantAClient = createS3ClientFromDescriptor(persistedTenantA);
  const getA = await tenantAClient.send(new GetObjectCommand({
    Bucket: WORKAPP_A_BUCKET,
    Key: 'hello.txt',
  }));
  expect(await streamToString(getA.Body as Readable)).toEqual('hello from tenant a');

  const tenants = await testSmartStorageInstance.listBucketTenants();
  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_A_BUCKET)).toEqual(true);
  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(true);

  const policyResponse = await adminClient.send(new GetBucketPolicyCommand({
    Bucket: POLICY_BUCKET,
  }));
  expect(policyResponse.Policy?.includes('TenantPolicyPersistence')).toEqual(true);
});

tap.test('deleteBucketTenant refuses buckets without tenant credentials', async () => {
  await expect(testSmartStorageInstance.deleteBucketTenant({
    bucketName: POLICY_BUCKET,
  })).rejects.toThrow();

  const headAfterRefusedDelete = await adminClient.send(new HeadBucketCommand({
    Bucket: POLICY_BUCKET,
  }));
  expect(headAfterRefusedDelete.$metadata.httpStatusCode).toEqual(200);
});

tap.test('deleteBucketTenant can revoke credentials and delete tenant buckets', async () => {
  const revokeOnlyTenant = await testSmartStorageInstance.createBucketTenant({
    bucketName: REVOKE_ONLY_BUCKET,
  });
  const revokeOnlyClient = createS3ClientFromDescriptor(revokeOnlyTenant);
  await revokeOnlyClient.send(new PutObjectCommand({
    Bucket: REVOKE_ONLY_BUCKET,
    Key: 'revoke-only.txt',
    Body: 'revocation target',
  }));

  await testSmartStorageInstance.deleteBucketTenant({
    bucketName: REVOKE_ONLY_BUCKET,
    accessKeyId: revokeOnlyTenant.accessKeyId,
  });

  await expect(revokeOnlyClient.send(new GetObjectCommand({
    Bucket: REVOKE_ONLY_BUCKET,
    Key: 'revoke-only.txt',
  }))).rejects.toThrow();

  const headAfterRevoke = await adminClient.send(new HeadBucketCommand({
    Bucket: REVOKE_ONLY_BUCKET,
  }));
  expect(headAfterRevoke.$metadata.httpStatusCode).toEqual(200);

  await testSmartStorageInstance.deleteBucketTenant({
    bucketName: WORKAPP_B_BUCKET,
  });
  await expect(tenantBClient.send(new GetObjectCommand({
    Bucket: WORKAPP_B_BUCKET,
    Key: 'other.txt',
  }))).rejects.toThrow();
  await expect(adminClient.send(new HeadBucketCommand({
    Bucket: WORKAPP_B_BUCKET,
  }))).rejects.toThrow();

  const tenants = await testSmartStorageInstance.listBucketTenants();
  expect(tenants.some((tenant) => tenant.bucketName === WORKAPP_B_BUCKET)).toEqual(false);
});

tap.test('teardown: stop storage server', async () => {
  await testSmartStorageInstance.stop();
});

export default tap.start();