Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bba0855218 | |||
| d437ffc226 | |||
| e36758f183 | |||
| adf45dce2d | |||
| eb232b6e8e |
32
changelog.md
32
changelog.md
@@ -1,5 +1,37 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-03-14 - 6.0.0 - BREAKING CHANGE(core)
|
||||||
|
Rebrand from smarts3 to smartstorage
|
||||||
|
|
||||||
|
- Package renamed from @push.rocks/smarts3 to @push.rocks/smartstorage
|
||||||
|
- Class renamed from Smarts3 to SmartStorage (no backward-compatible re-export)
|
||||||
|
- Interface renamed from ISmarts3Config to ISmartStorageConfig
|
||||||
|
- Method renamed from getS3Descriptor to getStorageDescriptor
|
||||||
|
- Rust binary renamed from rusts3 to ruststorage
|
||||||
|
- Rust types renamed: S3Error→StorageError, S3Action→StorageAction, S3Config→SmartStorageConfig, S3Server→StorageServer
|
||||||
|
- On-disk file extension changed from ._S3_object to ._storage_object (BREAKING for existing stored data)
|
||||||
|
- Default credentials changed from S3RVER to STORAGE
|
||||||
|
- All internal S3 branding removed; AWS S3 protocol compatibility (IAM actions, ARNs, SigV4) fully maintained
|
||||||
|
|
||||||
|
## 2026-02-17 - 5.3.0 - feat(auth)
|
||||||
|
add AWS SigV4 authentication and bucket policy support
|
||||||
|
|
||||||
|
- Implement AWS SigV4 full verification (constant-time comparison, 15-minute clock skew enforcement) and expose default signing region (server.region = 'us-east-1').
|
||||||
|
- Add IAM-style bucket policy engine with Put/Get/Delete policy APIs (GetBucketPolicy/PutBucketPolicy/DeleteBucketPolicy), wildcard action/resource matching, Allow/Deny evaluation, and on-disk persistence under .policies/{bucket}.policy.json.
|
||||||
|
- Documentation and README expanded with policy usage, examples, API table entries, and notes about policy CRUD and behavior for anonymous/authenticated requests.
|
||||||
|
- Rust code refactors: simplify storage/server result structs and multipart handling (removed several unused size/key/bucket fields), remove S3Error::to_response and error_xml helpers, and other internal cleanup to support new auth/policy features.
|
||||||
|
|
||||||
|
## 2026-02-17 - 5.2.0 - feat(auth,policy)
|
||||||
|
add AWS SigV4 authentication and S3 bucket policy support
|
||||||
|
|
||||||
|
- Implemented real AWS SigV4 verification (HMAC-SHA256), including x-amz-date handling, clock skew enforcement and constant-time signature comparison
|
||||||
|
- Added bucket policy model, validator and evaluation engine (Deny > Allow > NoOpinion) with a PolicyStore (RwLock cache + disk-backed .policies/*.policy.json)
|
||||||
|
- Integrated action resolution and auth+policy pipeline into the HTTP server: authorization checks run per-request, anonymous requests are denied by default, ListAllMyBuckets requires authentication
|
||||||
|
- Added bucket policy CRUD handlers via ?policy query parameter (GET/PUT/DELETE) and cleanup of policies on bucket deletion
|
||||||
|
- Storage and config updates: created .policies dir and policy path helpers; default region added to server config (TS + Rust)
|
||||||
|
- Added comprehensive tests for auth and policy behavior (policy CRUD, evaluation, per-action enforcement, auth integration)
|
||||||
|
- Updated Rust dependencies and Cargo.toml/Cargo.lock to include hmac, sha2, hex, subtle, cpufeatures
|
||||||
|
|
||||||
## 2026-02-13 - 5.1.1 - fix(smarts3)
|
## 2026-02-13 - 5.1.1 - fix(smarts3)
|
||||||
replace TypeScript server with Rust-powered core and IPC bridge
|
replace TypeScript server with Rust-powered core and IPC bridge
|
||||||
|
|
||||||
|
|||||||
@@ -10,14 +10,15 @@
|
|||||||
"module": {
|
"module": {
|
||||||
"githost": "code.foss.global",
|
"githost": "code.foss.global",
|
||||||
"gitscope": "push.rocks",
|
"gitscope": "push.rocks",
|
||||||
"gitrepo": "smarts3",
|
"gitrepo": "smartstorage",
|
||||||
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
|
"description": "A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.",
|
||||||
"npmPackagename": "@push.rocks/smarts3",
|
"npmPackagename": "@push.rocks/smartstorage",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"projectDomain": "push.rocks",
|
"projectDomain": "push.rocks",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"S3 Mock Server",
|
"smartstorage",
|
||||||
"Local S3",
|
"S3 Compatible",
|
||||||
|
"Local Storage Server",
|
||||||
"Node.js",
|
"Node.js",
|
||||||
"TypeScript",
|
"TypeScript",
|
||||||
"Local Development",
|
"Local Development",
|
||||||
@@ -26,8 +27,8 @@
|
|||||||
"File Storage",
|
"File Storage",
|
||||||
"AWS S3 Compatibility",
|
"AWS S3 Compatibility",
|
||||||
"Development Tool",
|
"Development Tool",
|
||||||
"S3 Endpoint",
|
"Storage Endpoint",
|
||||||
"S3 Simulation",
|
"Storage Simulation",
|
||||||
"Bucket Management",
|
"Bucket Management",
|
||||||
"File Upload",
|
"File Upload",
|
||||||
"CI/CD Integration",
|
"CI/CD Integration",
|
||||||
|
|||||||
21
package.json
21
package.json
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"name": "@push.rocks/smarts3",
|
"name": "@push.rocks/smartstorage",
|
||||||
"version": "5.1.1",
|
"version": "6.0.0",
|
||||||
"private": false,
|
"private": false,
|
||||||
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
|
"description": "A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
"typings": "dist_ts/index.d.ts",
|
"typings": "dist_ts/index.d.ts",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
@@ -45,8 +45,9 @@
|
|||||||
"@tsclass/tsclass": "^9.3.0"
|
"@tsclass/tsclass": "^9.3.0"
|
||||||
},
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"S3 Mock Server",
|
"smartstorage",
|
||||||
"Local S3",
|
"S3 Compatible",
|
||||||
|
"Local Storage Server",
|
||||||
"Node.js",
|
"Node.js",
|
||||||
"TypeScript",
|
"TypeScript",
|
||||||
"Local Development",
|
"Local Development",
|
||||||
@@ -55,20 +56,20 @@
|
|||||||
"File Storage",
|
"File Storage",
|
||||||
"AWS S3 Compatibility",
|
"AWS S3 Compatibility",
|
||||||
"Development Tool",
|
"Development Tool",
|
||||||
"S3 Endpoint",
|
"Storage Endpoint",
|
||||||
"S3 Simulation",
|
"Storage Simulation",
|
||||||
"Bucket Management",
|
"Bucket Management",
|
||||||
"File Upload",
|
"File Upload",
|
||||||
"CI/CD Integration",
|
"CI/CD Integration",
|
||||||
"Developer Onboarding"
|
"Developer Onboarding"
|
||||||
],
|
],
|
||||||
"homepage": "https://code.foss.global/push.rocks/smarts3#readme",
|
"homepage": "https://code.foss.global/push.rocks/smartstorage#readme",
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://code.foss.global/push.rocks/smarts3.git"
|
"url": "ssh://git@code.foss.global:29419/push.rocks/smartstorage.git"
|
||||||
},
|
},
|
||||||
"bugs": {
|
"bugs": {
|
||||||
"url": "https://code.foss.global/push.rocks/smarts3/issues"
|
"url": "https://code.foss.global/push.rocks/smartstorage/issues"
|
||||||
},
|
},
|
||||||
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748",
|
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748",
|
||||||
"pnpm": {
|
"pnpm": {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Production-Readiness Plan for smarts3
|
# Production-Readiness Plan for smartstorage
|
||||||
|
|
||||||
**Goal:** Make smarts3 production-ready as a MinIO alternative for use cases where:
|
**Goal:** Make smartstorage production-ready as a MinIO alternative for use cases where:
|
||||||
- Running MinIO is out of scope
|
- Running MinIO is out of scope
|
||||||
- You have a program written for S3 and want to use the local filesystem
|
- You have a program written for S3 and want to use the local filesystem
|
||||||
- You need a lightweight, zero-dependency S3-compatible server
|
- You need a lightweight, zero-dependency S3-compatible server
|
||||||
@@ -31,7 +31,7 @@
|
|||||||
|
|
||||||
### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY**
|
### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY**
|
||||||
|
|
||||||
**Why:** Essential for uploading files >5MB efficiently. Without this, smarts3 can't handle real-world production workloads.
|
**Why:** Essential for uploading files >5MB efficiently. Without this, smartstorage can't handle real-world production workloads.
|
||||||
|
|
||||||
**Implementation Required:**
|
**Implementation Required:**
|
||||||
- `POST /:bucket/:key?uploads` - CreateMultipartUpload
|
- `POST /:bucket/:key?uploads` - CreateMultipartUpload
|
||||||
@@ -46,13 +46,13 @@
|
|||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/controllers/multipart.controller.ts` (new)
|
- `ts/controllers/multipart.controller.ts` (new)
|
||||||
- `ts/classes/filesystem-store.ts` (add multipart methods)
|
- `ts/classes/filesystem-store.ts` (add multipart methods)
|
||||||
- `ts/classes/smarts3-server.ts` (add multipart routes)
|
- `ts/classes/smartstorage-server.ts` (add multipart routes)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 2. Configurable Authentication 🔐
|
### 2. Configurable Authentication 🔐
|
||||||
|
|
||||||
**Why:** Currently hardcoded credentials ('S3RVER'/'S3RVER'). Production needs custom credentials.
|
**Why:** Currently hardcoded credentials ('STORAGE'/'STORAGE'). Production needs custom credentials.
|
||||||
|
|
||||||
**Implementation Required:**
|
**Implementation Required:**
|
||||||
- Support custom access keys and secrets via configuration
|
- Support custom access keys and secrets via configuration
|
||||||
@@ -75,7 +75,7 @@ interface IAuthConfig {
|
|||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/auth-middleware.ts` (new)
|
- `ts/classes/auth-middleware.ts` (new)
|
||||||
- `ts/classes/signature-validator.ts` (new)
|
- `ts/classes/signature-validator.ts` (new)
|
||||||
- `ts/classes/smarts3-server.ts` (integrate auth middleware)
|
- `ts/classes/smartstorage-server.ts` (integrate auth middleware)
|
||||||
- `ts/index.ts` (add auth config options)
|
- `ts/index.ts` (add auth config options)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -105,7 +105,7 @@ interface ICorsConfig {
|
|||||||
|
|
||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/cors-middleware.ts` (new)
|
- `ts/classes/cors-middleware.ts` (new)
|
||||||
- `ts/classes/smarts3-server.ts` (integrate CORS middleware)
|
- `ts/classes/smartstorage-server.ts` (integrate CORS middleware)
|
||||||
- `ts/index.ts` (add CORS config options)
|
- `ts/index.ts` (add CORS config options)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -131,7 +131,7 @@ interface ISslConfig {
|
|||||||
```
|
```
|
||||||
|
|
||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/smarts3-server.ts` (add HTTPS server creation)
|
- `ts/classes/smartstorage-server.ts` (add HTTPS server creation)
|
||||||
- `ts/index.ts` (add SSL config options)
|
- `ts/index.ts` (add SSL config options)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -147,7 +147,7 @@ interface ISslConfig {
|
|||||||
- Sensible production defaults
|
- Sensible production defaults
|
||||||
- Example configurations for common use cases
|
- Example configurations for common use cases
|
||||||
|
|
||||||
**Configuration File Example (`smarts3.config.json`):**
|
**Configuration File Example (`smartstorage.config.json`):**
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"server": {
|
"server": {
|
||||||
@@ -220,7 +220,7 @@ interface ISslConfig {
|
|||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?)
|
- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?)
|
||||||
- `ts/classes/access-logger-middleware.ts` (new)
|
- `ts/classes/access-logger-middleware.ts` (new)
|
||||||
- `ts/classes/smarts3-server.ts` (replace console.log with logger)
|
- `ts/classes/smartstorage-server.ts` (replace console.log with logger)
|
||||||
- All controller files (use structured logging)
|
- All controller files (use structured logging)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -238,7 +238,7 @@ interface ISslConfig {
|
|||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/controllers/health.controller.ts` (new)
|
- `ts/controllers/health.controller.ts` (new)
|
||||||
- `ts/classes/metrics-collector.ts` (new)
|
- `ts/classes/metrics-collector.ts` (new)
|
||||||
- `ts/classes/smarts3-server.ts` (add health routes)
|
- `ts/classes/smartstorage-server.ts` (add health routes)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -266,7 +266,7 @@ interface ISslConfig {
|
|||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/validation-middleware.ts` (new)
|
- `ts/classes/validation-middleware.ts` (new)
|
||||||
- `ts/utils/validators.ts` (new)
|
- `ts/utils/validators.ts` (new)
|
||||||
- `ts/classes/smarts3-server.ts` (integrate validation middleware)
|
- `ts/classes/smartstorage-server.ts` (integrate validation middleware)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -291,7 +291,7 @@ interface ISslConfig {
|
|||||||
- SIGTERM/SIGINT handling
|
- SIGTERM/SIGINT handling
|
||||||
|
|
||||||
**Files to Create/Modify:**
|
**Files to Create/Modify:**
|
||||||
- `ts/classes/smarts3-server.ts` (add graceful shutdown logic)
|
- `ts/classes/smartstorage-server.ts` (add graceful shutdown logic)
|
||||||
- `ts/index.ts` (add signal handlers)
|
- `ts/index.ts` (add signal handlers)
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -336,7 +336,7 @@ interface ISslConfig {
|
|||||||
4. ✅ Production configuration system
|
4. ✅ Production configuration system
|
||||||
5. ✅ Production logging
|
5. ✅ Production logging
|
||||||
|
|
||||||
**Outcome:** smarts3 can handle real production workloads
|
**Outcome:** smartstorage can handle real production workloads
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -350,7 +350,7 @@ interface ISslConfig {
|
|||||||
9. ✅ Graceful shutdown
|
9. ✅ Graceful shutdown
|
||||||
10. ✅ Batch operations
|
10. ✅ Batch operations
|
||||||
|
|
||||||
**Outcome:** smarts3 is operationally mature
|
**Outcome:** smartstorage is operationally mature
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -363,7 +363,7 @@ interface ISslConfig {
|
|||||||
13. ✅ Comprehensive test suite
|
13. ✅ Comprehensive test suite
|
||||||
14. ✅ Documentation updates
|
14. ✅ Documentation updates
|
||||||
|
|
||||||
**Outcome:** smarts3 has broad S3 API compatibility
|
**Outcome:** smartstorage has broad S3 API compatibility
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -375,7 +375,7 @@ interface ISslConfig {
|
|||||||
16. ✅ Performance optimization
|
16. ✅ Performance optimization
|
||||||
17. ✅ Advanced features based on user feedback
|
17. ✅ Advanced features based on user feedback
|
||||||
|
|
||||||
**Outcome:** smarts3 is a complete MinIO alternative
|
**Outcome:** smartstorage is a complete MinIO alternative
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -392,7 +392,7 @@ interface ISslConfig {
|
|||||||
|
|
||||||
## 🎯 Target Use Cases
|
## 🎯 Target Use Cases
|
||||||
|
|
||||||
**With this plan implemented, smarts3 will be a solid MinIO alternative for:**
|
**With this plan implemented, smartstorage will be a solid MinIO alternative for:**
|
||||||
|
|
||||||
✅ **Local S3 development** - Fast, simple, no Docker required
|
✅ **Local S3 development** - Fast, simple, no Docker required
|
||||||
✅ **Testing S3 integrations** - Reliable, repeatable tests
|
✅ **Testing S3 integrations** - Reliable, repeatable tests
|
||||||
|
|||||||
@@ -1,13 +1,14 @@
|
|||||||
# Project Hints for smarts3
|
# Project Hints for smartstorage
|
||||||
|
|
||||||
## Current State (v6.0.0-dev)
|
## Current State (v6.0.0)
|
||||||
|
|
||||||
- **Rust-powered S3 server** via `@push.rocks/smartrust` IPC bridge
|
- **Rust-powered S3-compatible storage server** via `@push.rocks/smartrust` IPC bridge
|
||||||
- High-performance: streaming I/O, zero-copy, backpressure, range seek
|
- High-performance: streaming I/O, zero-copy, backpressure, range seek
|
||||||
- TypeScript is thin IPC wrapper; all HTTP/storage/routing in Rust binary `rusts3`
|
- TypeScript is thin IPC wrapper; all HTTP/storage/routing in Rust binary `ruststorage`
|
||||||
- Full S3 compatibility: PUT, GET, HEAD, DELETE for objects and buckets
|
- Full S3 compatibility: PUT, GET, HEAD, DELETE for objects and buckets
|
||||||
- Multipart upload support (streaming, no OOM)
|
- Multipart upload support (streaming, no OOM)
|
||||||
- Authentication (AWS v2/v4 signature key extraction)
|
- **Real AWS SigV4 authentication** (cryptographic signature verification)
|
||||||
|
- **Bucket policies** (AWS/MinIO-compatible JSON policies, public access support)
|
||||||
- CORS support
|
- CORS support
|
||||||
- ListBuckets, ListObjects (v1/v2), CopyObject
|
- ListBuckets, ListObjects (v1/v2), CopyObject
|
||||||
|
|
||||||
@@ -15,35 +16,39 @@
|
|||||||
|
|
||||||
### Rust Binary (`rust/src/`)
|
### Rust Binary (`rust/src/`)
|
||||||
- `main.rs` - Clap CLI, management mode entry
|
- `main.rs` - Clap CLI, management mode entry
|
||||||
- `config.rs` - Serde config structs matching TS interfaces
|
- `config.rs` - Serde config structs matching TS interfaces (includes `region`)
|
||||||
- `management.rs` - IPC loop (newline-delimited JSON over stdin/stdout)
|
- `management.rs` - IPC loop (newline-delimited JSON over stdin/stdout)
|
||||||
- `server.rs` - hyper 1.x HTTP server, routing, CORS, auth, all S3 handlers
|
- `server.rs` - hyper 1.x HTTP server, routing, CORS, auth+policy pipeline, all S3-compatible handlers
|
||||||
- `storage.rs` - FileStore: filesystem-backed storage, multipart manager
|
- `storage.rs` - FileStore: filesystem-backed storage, multipart manager, `.policies/` dir
|
||||||
- `xml_response.rs` - S3 XML response builders
|
- `xml_response.rs` - S3-compatible XML response builders
|
||||||
- `s3_error.rs` - S3 error codes with HTTP status mapping
|
- `error.rs` - StorageError codes with HTTP status mapping
|
||||||
|
- `auth.rs` - AWS SigV4 signature verification (HMAC-SHA256, clock skew, constant-time compare)
|
||||||
|
- `action.rs` - StorageAction enum + request-to-IAM-action resolver + RequestContext
|
||||||
|
- `policy.rs` - BucketPolicy model, evaluation engine (Deny > Allow > NoOpinion), PolicyStore (RwLock cache + disk)
|
||||||
|
|
||||||
### TypeScript Bridge (`ts/`)
|
### TypeScript Bridge (`ts/`)
|
||||||
- `ts/index.ts` - Smarts3 class with RustBridge<TRustS3Commands>
|
- `ts/index.ts` - SmartStorage class with RustBridge<TRustStorageCommands>
|
||||||
- `ts/plugins.ts` - path, smartpath, RustBridge, tsclass
|
- `ts/plugins.ts` - path, smartpath, RustBridge, tsclass
|
||||||
- `ts/paths.ts` - packageDir, bucketsDir defaults
|
- `ts/paths.ts` - packageDir, bucketsDir defaults
|
||||||
|
|
||||||
### IPC Commands
|
### IPC Commands
|
||||||
| Command | Params | Action |
|
| Command | Params | Action |
|
||||||
|---------|--------|--------|
|
|---------|--------|--------|
|
||||||
| `start` | `{ config: ISmarts3Config }` | Init storage + HTTP server |
|
| `start` | `{ config: ISmartStorageConfig }` | Init storage + HTTP server |
|
||||||
| `stop` | `{}` | Graceful shutdown |
|
| `stop` | `{}` | Graceful shutdown |
|
||||||
| `createBucket` | `{ name: string }` | Create bucket directory |
|
| `createBucket` | `{ name: string }` | Create bucket directory |
|
||||||
|
|
||||||
### Storage Layout (backward-compatible)
|
### Storage Layout
|
||||||
- Objects: `{root}/{bucket}/{key}._S3_object`
|
- Objects: `{root}/{bucket}/{key}._storage_object`
|
||||||
- Metadata: `{root}/{bucket}/{key}._S3_object.metadata.json`
|
- Metadata: `{root}/{bucket}/{key}._storage_object.metadata.json`
|
||||||
- MD5: `{root}/{bucket}/{key}._S3_object.md5`
|
- MD5: `{root}/{bucket}/{key}._storage_object.md5`
|
||||||
- Multipart: `{root}/.multipart/{upload_id}/part-{N}`
|
- Multipart: `{root}/.multipart/{upload_id}/part-{N}`
|
||||||
|
- Policies: `{root}/.policies/{bucket}.policy.json`
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
- `pnpm build` runs `tsrust && tsbuild --web --allowimplicitany`
|
- `pnpm build` runs `tsrust && tsbuild --web --allowimplicitany`
|
||||||
- `tsrust` compiles Rust to `dist_rust/rusts3`
|
- `tsrust` compiles Rust to `dist_rust/ruststorage`
|
||||||
- Targets: linux_amd64, linux_arm64 (configured in npmextra.json)
|
- Targets: linux_amd64, linux_arm64 (configured in npmextra.json)
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
@@ -55,6 +60,10 @@
|
|||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
- `test/test.aws-sdk.node.ts` - AWS SDK v3 compatibility (10 tests)
|
- `test/test.aws-sdk.node.ts` - AWS SDK v3 compatibility (10 tests, auth disabled, port 3337)
|
||||||
|
- `test/test.auth.node.ts` - Auth + bucket policy integration (20 tests, auth enabled, port 3344)
|
||||||
|
- `test/test.policy-crud.node.ts` - Policy API CRUD + validation edge cases (17 tests, port 3345)
|
||||||
|
- `test/test.policy-eval.node.ts` - Policy evaluation: principals, actions, resources, deny-vs-allow (22 tests, port 3346)
|
||||||
|
- `test/test.policy-actions.node.ts` - Per-action policy enforcement (15 tests, port 3347)
|
||||||
- `test/test.ts` - SmartBucket integration (3 tests)
|
- `test/test.ts` - SmartBucket integration (3 tests)
|
||||||
- Run: `pnpm test` or `tstest test/test.aws-sdk.node.ts --verbose`
|
- Run: `pnpm test` or `tstest test/test.aws-sdk.node.ts --verbose`
|
||||||
|
|||||||
199
readme.md
199
readme.md
@@ -1,78 +1,81 @@
|
|||||||
# @push.rocks/smarts3 🚀
|
# @push.rocks/smartstorage
|
||||||
|
|
||||||
A high-performance, S3-compatible local server powered by a **Rust core** with a clean TypeScript API. Drop-in replacement for AWS S3 during development and testing — no cloud, no Docker, no MinIO. Just `npm install` and go.
|
A high-performance, S3-compatible local storage server powered by a **Rust core** with a clean TypeScript API. Drop-in replacement for AWS S3 during development and testing — no cloud, no Docker, no MinIO. Just `npm install` and go.
|
||||||
|
|
||||||
## Issue Reporting and Security
|
## Issue Reporting and Security
|
||||||
|
|
||||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||||
|
|
||||||
## 🌟 Why smarts3?
|
## Why smartstorage?
|
||||||
|
|
||||||
| Feature | smarts3 | MinIO | s3rver |
|
| Feature | smartstorage | MinIO | s3rver |
|
||||||
|---------|---------|-------|--------|
|
|---------|-------------|-------|--------|
|
||||||
| Install | `pnpm add` | Docker / binary | `npm install` |
|
| Install | `pnpm add` | Docker / binary | `npm install` |
|
||||||
| Startup time | ~20ms | seconds | ~200ms |
|
| Startup time | ~20ms | seconds | ~200ms |
|
||||||
| Large file uploads | ✅ Streaming, zero-copy | ✅ | ❌ OOM risk |
|
| Large file uploads | Streaming, zero-copy | Yes | OOM risk |
|
||||||
| Range requests | ✅ Seek-based | ✅ | ❌ Full read |
|
| Range requests | Seek-based | Yes | Full read |
|
||||||
| Language | Rust + TypeScript | Go | JavaScript |
|
| Language | Rust + TypeScript | Go | JavaScript |
|
||||||
| Multipart uploads | ✅ Full support | ✅ | ❌ |
|
| Multipart uploads | Full support | Yes | No |
|
||||||
| Auth | AWS v2/v4 key extraction | Full IAM | Basic |
|
| Auth | AWS SigV4 (full verification) | Full IAM | Basic |
|
||||||
|
| Bucket policies | IAM-style evaluation | Yes | No |
|
||||||
|
|
||||||
### Core Features
|
### Core Features
|
||||||
|
|
||||||
- ⚡ **Rust-powered HTTP server** — hyper 1.x with streaming I/O, zero-copy, backpressure
|
- **Rust-powered HTTP server** — hyper 1.x with streaming I/O, zero-copy, backpressure
|
||||||
- 🔄 **Full S3 API compatibility** — works with AWS SDK v3, SmartBucket, any S3 client
|
- **Full S3-compatible API** — works with AWS SDK v3, SmartBucket, any S3 client
|
||||||
- 📂 **Filesystem-backed storage** — buckets map to directories, objects to files
|
- **Filesystem-backed storage** — buckets map to directories, objects to files
|
||||||
- 📤 **Streaming multipart uploads** — large files without memory pressure
|
- **Streaming multipart uploads** — large files without memory pressure
|
||||||
- 🎯 **Byte-range requests** — `seek()` directly to the requested byte offset
|
- **Byte-range requests** — `seek()` directly to the requested byte offset
|
||||||
- 🔐 **Authentication** — AWS v2/v4 signature key extraction
|
- **AWS SigV4 authentication** — full signature verification with constant-time comparison and 15-min clock skew enforcement
|
||||||
- 🌐 **CORS middleware** — configurable cross-origin support
|
- **Bucket policies** — IAM-style JSON policies with Allow/Deny evaluation, wildcard matching, and anonymous access support
|
||||||
- 📊 **Structured logging** — tracing-based, error through debug levels
|
- **CORS middleware** — configurable cross-origin support
|
||||||
- 🧹 **Clean slate mode** — wipe storage on startup for test isolation
|
- **Structured logging** — tracing-based, error through debug levels
|
||||||
- 🧪 **Test-first design** — start/stop in milliseconds, no port conflicts
|
- **Clean slate mode** — wipe storage on startup for test isolation
|
||||||
|
- **Test-first design** — start/stop in milliseconds, no port conflicts
|
||||||
|
|
||||||
## 📦 Installation
|
## Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pnpm add @push.rocks/smarts3 -D
|
pnpm add @push.rocks/smartstorage -D
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Note:** The package ships with precompiled Rust binaries for `linux_amd64` and `linux_arm64`. No Rust toolchain needed on your machine.
|
> **Note:** The package ships with precompiled Rust binaries for `linux_amd64` and `linux_arm64`. No Rust toolchain needed on your machine.
|
||||||
|
|
||||||
## 🚀 Quick Start
|
## Quick Start
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Smarts3 } from '@push.rocks/smarts3';
|
import { SmartStorage } from '@push.rocks/smartstorage';
|
||||||
|
|
||||||
// Start a local S3 server
|
// Start a local S3-compatible storage server
|
||||||
const s3 = await Smarts3.createAndStart({
|
const storage = await SmartStorage.createAndStart({
|
||||||
server: { port: 3000 },
|
server: { port: 3000 },
|
||||||
storage: { cleanSlate: true },
|
storage: { cleanSlate: true },
|
||||||
});
|
});
|
||||||
|
|
||||||
// Create a bucket
|
// Create a bucket
|
||||||
await s3.createBucket('my-bucket');
|
await storage.createBucket('my-bucket');
|
||||||
|
|
||||||
// Get connection details for any S3 client
|
// Get connection details for any S3 client
|
||||||
const descriptor = await s3.getS3Descriptor();
|
const descriptor = await storage.getStorageDescriptor();
|
||||||
// → { endpoint: 'localhost', port: 3000, accessKey: 'S3RVER', accessSecret: 'S3RVER', useSsl: false }
|
// → { endpoint: 'localhost', port: 3000, accessKey: 'STORAGE', accessSecret: 'STORAGE', useSsl: false }
|
||||||
|
|
||||||
// When done
|
// When done
|
||||||
await s3.stop();
|
await storage.stop();
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📖 Configuration
|
## Configuration
|
||||||
|
|
||||||
All config fields are optional — sensible defaults are applied automatically.
|
All config fields are optional — sensible defaults are applied automatically.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Smarts3, ISmarts3Config } from '@push.rocks/smarts3';
|
import { SmartStorage, ISmartStorageConfig } from '@push.rocks/smartstorage';
|
||||||
|
|
||||||
const config: ISmarts3Config = {
|
const config: ISmartStorageConfig = {
|
||||||
server: {
|
server: {
|
||||||
port: 3000, // Default: 3000
|
port: 3000, // Default: 3000
|
||||||
address: '0.0.0.0', // Default: '0.0.0.0'
|
address: '0.0.0.0', // Default: '0.0.0.0'
|
||||||
silent: false, // Default: false
|
silent: false, // Default: false
|
||||||
|
region: 'us-east-1', // Default: 'us-east-1' — used for SigV4 signing
|
||||||
},
|
},
|
||||||
storage: {
|
storage: {
|
||||||
directory: './my-data', // Default: .nogit/bucketsDir
|
directory: './my-data', // Default: .nogit/bucketsDir
|
||||||
@@ -110,14 +113,14 @@ const config: ISmarts3Config = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const s3 = await Smarts3.createAndStart(config);
|
const storage = await SmartStorage.createAndStart(config);
|
||||||
```
|
```
|
||||||
|
|
||||||
### Common Configurations
|
### Common Configurations
|
||||||
|
|
||||||
**CI/CD testing** — silent, clean, fast:
|
**CI/CD testing** — silent, clean, fast:
|
||||||
```typescript
|
```typescript
|
||||||
const s3 = await Smarts3.createAndStart({
|
const storage = await SmartStorage.createAndStart({
|
||||||
server: { port: 9999, silent: true },
|
server: { port: 9999, silent: true },
|
||||||
storage: { cleanSlate: true },
|
storage: { cleanSlate: true },
|
||||||
});
|
});
|
||||||
@@ -125,7 +128,7 @@ const s3 = await Smarts3.createAndStart({
|
|||||||
|
|
||||||
**Auth enabled:**
|
**Auth enabled:**
|
||||||
```typescript
|
```typescript
|
||||||
const s3 = await Smarts3.createAndStart({
|
const storage = await SmartStorage.createAndStart({
|
||||||
auth: {
|
auth: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
credentials: [{ accessKeyId: 'test', secretAccessKey: 'test123' }],
|
credentials: [{ accessKeyId: 'test', secretAccessKey: 'test123' }],
|
||||||
@@ -135,7 +138,7 @@ const s3 = await Smarts3.createAndStart({
|
|||||||
|
|
||||||
**CORS for local web dev:**
|
**CORS for local web dev:**
|
||||||
```typescript
|
```typescript
|
||||||
const s3 = await Smarts3.createAndStart({
|
const storage = await SmartStorage.createAndStart({
|
||||||
cors: {
|
cors: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
allowedOrigins: ['http://localhost:5173'],
|
allowedOrigins: ['http://localhost:5173'],
|
||||||
@@ -144,12 +147,12 @@ const s3 = await Smarts3.createAndStart({
|
|||||||
});
|
});
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📤 Usage with AWS SDK v3
|
## Usage with AWS SDK v3
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
||||||
|
|
||||||
const descriptor = await s3.getS3Descriptor();
|
const descriptor = await storage.getStorageDescriptor();
|
||||||
|
|
||||||
const client = new S3Client({
|
const client = new S3Client({
|
||||||
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
||||||
@@ -158,14 +161,14 @@ const client = new S3Client({
|
|||||||
accessKeyId: descriptor.accessKey,
|
accessKeyId: descriptor.accessKey,
|
||||||
secretAccessKey: descriptor.accessSecret,
|
secretAccessKey: descriptor.accessSecret,
|
||||||
},
|
},
|
||||||
forcePathStyle: true, // Required for path-style S3
|
forcePathStyle: true, // Required for path-style access
|
||||||
});
|
});
|
||||||
|
|
||||||
// Upload
|
// Upload
|
||||||
await client.send(new PutObjectCommand({
|
await client.send(new PutObjectCommand({
|
||||||
Bucket: 'my-bucket',
|
Bucket: 'my-bucket',
|
||||||
Key: 'hello.txt',
|
Key: 'hello.txt',
|
||||||
Body: 'Hello, S3!',
|
Body: 'Hello, Storage!',
|
||||||
ContentType: 'text/plain',
|
ContentType: 'text/plain',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -174,7 +177,7 @@ const { Body } = await client.send(new GetObjectCommand({
|
|||||||
Bucket: 'my-bucket',
|
Bucket: 'my-bucket',
|
||||||
Key: 'hello.txt',
|
Key: 'hello.txt',
|
||||||
}));
|
}));
|
||||||
const content = await Body.transformToString(); // "Hello, S3!"
|
const content = await Body.transformToString(); // "Hello, Storage!"
|
||||||
|
|
||||||
// Delete
|
// Delete
|
||||||
await client.send(new DeleteObjectCommand({
|
await client.send(new DeleteObjectCommand({
|
||||||
@@ -183,12 +186,12 @@ await client.send(new DeleteObjectCommand({
|
|||||||
}));
|
}));
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🪣 Usage with SmartBucket
|
## Usage with SmartBucket
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { SmartBucket } from '@push.rocks/smartbucket';
|
import { SmartBucket } from '@push.rocks/smartbucket';
|
||||||
|
|
||||||
const smartbucket = new SmartBucket(await s3.getS3Descriptor());
|
const smartbucket = new SmartBucket(await storage.getStorageDescriptor());
|
||||||
const bucket = await smartbucket.createBucket('my-bucket');
|
const bucket = await smartbucket.createBucket('my-bucket');
|
||||||
const dir = await bucket.getBaseDirectory();
|
const dir = await bucket.getBaseDirectory();
|
||||||
|
|
||||||
@@ -202,9 +205,9 @@ const content = await dir.fastGet('docs/readme.txt');
|
|||||||
const files = await dir.listFiles();
|
const files = await dir.listFiles();
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📤 Multipart Uploads
|
## Multipart Uploads
|
||||||
|
|
||||||
For files larger than 5 MB, use multipart uploads. smarts3 handles them with **streaming I/O** — parts are written directly to disk, never buffered in memory.
|
For files larger than 5 MB, use multipart uploads. smartstorage handles them with **streaming I/O** — parts are written directly to disk, never buffered in memory.
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import {
|
import {
|
||||||
@@ -241,38 +244,88 @@ await client.send(new CompleteMultipartUploadCommand({
|
|||||||
}));
|
}));
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🧪 Testing Integration
|
## Bucket Policies
|
||||||
|
|
||||||
|
smartstorage supports AWS-style bucket policies for fine-grained access control. Policies use the same IAM JSON format as real S3 — so you can develop and test your policy logic locally before deploying.
|
||||||
|
|
||||||
|
When `auth.enabled` is `true`, the auth pipeline works as follows:
|
||||||
|
1. **Authenticate** — verify the AWS SigV4 signature (anonymous requests skip this step)
|
||||||
|
2. **Authorize** — evaluate bucket policies against the request action, resource, and caller identity
|
||||||
|
3. **Default** — authenticated users get full access; anonymous requests are denied unless a policy explicitly allows them
|
||||||
|
|
||||||
|
### Setting a Bucket Policy
|
||||||
|
|
||||||
|
Use the S3 `PutBucketPolicy` API (or any S3 client that supports it):
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
import { Smarts3 } from '@push.rocks/smarts3';
|
import { PutBucketPolicyCommand } from '@aws-sdk/client-s3';
|
||||||
|
|
||||||
|
// Allow anonymous read access to all objects in a bucket
|
||||||
|
await client.send(new PutBucketPolicyCommand({
|
||||||
|
Bucket: 'public-assets',
|
||||||
|
Policy: JSON.stringify({
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [{
|
||||||
|
Sid: 'PublicRead',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject'],
|
||||||
|
Resource: ['arn:aws:s3:::public-assets/*'],
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
```
|
||||||
|
|
||||||
|
### Policy Features
|
||||||
|
|
||||||
|
- **Effect**: `Allow` and `Deny` (explicit Deny always wins)
|
||||||
|
- **Principal**: `"*"` (everyone) or `{ "AWS": ["arn:..."] }` for specific identities
|
||||||
|
- **Action**: IAM-style actions like `s3:GetObject`, `s3:PutObject`, `s3:*`, or prefix wildcards like `s3:Get*`
|
||||||
|
- **Resource**: ARN patterns with `*` and `?` wildcards (e.g. `arn:aws:s3:::my-bucket/*`)
|
||||||
|
- **Persistence**: Policies survive server restarts — stored as JSON on disk alongside your data
|
||||||
|
|
||||||
|
### Policy CRUD Operations
|
||||||
|
|
||||||
|
| Operation | AWS SDK Command | HTTP |
|
||||||
|
|-----------|----------------|------|
|
||||||
|
| Get policy | `GetBucketPolicyCommand` | `GET /{bucket}?policy` |
|
||||||
|
| Set policy | `PutBucketPolicyCommand` | `PUT /{bucket}?policy` |
|
||||||
|
| Delete policy | `DeleteBucketPolicyCommand` | `DELETE /{bucket}?policy` |
|
||||||
|
|
||||||
|
Deleting a bucket automatically removes its associated policy.
|
||||||
|
|
||||||
|
## Testing Integration
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { SmartStorage } from '@push.rocks/smartstorage';
|
||||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||||
|
|
||||||
let s3: Smarts3;
|
let storage: SmartStorage;
|
||||||
|
|
||||||
tap.test('setup', async () => {
|
tap.test('setup', async () => {
|
||||||
s3 = await Smarts3.createAndStart({
|
storage = await SmartStorage.createAndStart({
|
||||||
server: { port: 4567, silent: true },
|
server: { port: 4567, silent: true },
|
||||||
storage: { cleanSlate: true },
|
storage: { cleanSlate: true },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('should store and retrieve objects', async () => {
|
tap.test('should store and retrieve objects', async () => {
|
||||||
await s3.createBucket('test');
|
await storage.createBucket('test');
|
||||||
// ... your test logic using AWS SDK or SmartBucket
|
// ... your test logic using AWS SDK or SmartBucket
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('teardown', async () => {
|
tap.test('teardown', async () => {
|
||||||
await s3.stop();
|
await storage.stop();
|
||||||
});
|
});
|
||||||
|
|
||||||
export default tap.start();
|
export default tap.start();
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔧 API Reference
|
## API Reference
|
||||||
|
|
||||||
### `Smarts3` Class
|
### `SmartStorage` Class
|
||||||
|
|
||||||
#### `static createAndStart(config?: ISmarts3Config): Promise<Smarts3>`
|
#### `static createAndStart(config?: ISmartStorageConfig): Promise<SmartStorage>`
|
||||||
|
|
||||||
Create and start a server in one call.
|
Create and start a server in one call.
|
||||||
|
|
||||||
@@ -286,11 +339,11 @@ Gracefully stop the server and kill the Rust process.
|
|||||||
|
|
||||||
#### `createBucket(name: string): Promise<{ name: string }>`
|
#### `createBucket(name: string): Promise<{ name: string }>`
|
||||||
|
|
||||||
Create an S3 bucket.
|
Create a storage bucket.
|
||||||
|
|
||||||
#### `getS3Descriptor(options?): Promise<IS3Descriptor>`
|
#### `getStorageDescriptor(options?): Promise<IS3Descriptor>`
|
||||||
|
|
||||||
Get connection details for S3 clients. Returns:
|
Get connection details for S3-compatible clients. Returns:
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
|-------|------|-------------|
|
|-------|------|-------------|
|
||||||
@@ -300,25 +353,26 @@ Get connection details for S3 clients. Returns:
|
|||||||
| `accessSecret` | `string` | Secret key from first configured credential |
|
| `accessSecret` | `string` | Secret key from first configured credential |
|
||||||
| `useSsl` | `boolean` | Always `false` (plain HTTP) |
|
| `useSsl` | `boolean` | Always `false` (plain HTTP) |
|
||||||
|
|
||||||
## 🏗️ Architecture
|
## Architecture
|
||||||
|
|
||||||
smarts3 uses a **hybrid Rust + TypeScript** architecture:
|
smartstorage uses a **hybrid Rust + TypeScript** architecture:
|
||||||
|
|
||||||
```
|
```
|
||||||
┌─────────────────────────────────┐
|
┌─────────────────────────────────┐
|
||||||
│ Your Code (AWS SDK, etc.) │
|
│ Your Code (AWS SDK, etc.) │
|
||||||
│ ↕ HTTP (localhost:3000) │
|
│ ↕ HTTP (localhost:3000) │
|
||||||
├─────────────────────────────────┤
|
├─────────────────────────────────┤
|
||||||
│ rusts3 binary (Rust) │
|
│ ruststorage binary (Rust) │
|
||||||
│ ├─ hyper 1.x HTTP server │
|
│ ├─ hyper 1.x HTTP server │
|
||||||
│ ├─ S3 path-style routing │
|
│ ├─ S3 path-style routing │
|
||||||
│ ├─ Streaming storage layer │
|
│ ├─ Streaming storage layer │
|
||||||
│ ├─ Multipart manager │
|
│ ├─ Multipart manager │
|
||||||
│ ├─ CORS / Auth middleware │
|
│ ├─ SigV4 auth + policy engine │
|
||||||
|
│ ├─ CORS middleware │
|
||||||
│ └─ S3 XML response builder │
|
│ └─ S3 XML response builder │
|
||||||
├─────────────────────────────────┤
|
├─────────────────────────────────┤
|
||||||
│ TypeScript (thin IPC wrapper) │
|
│ TypeScript (thin IPC wrapper) │
|
||||||
│ ├─ Smarts3 class │
|
│ ├─ SmartStorage class │
|
||||||
│ ├─ RustBridge (stdin/stdout) │
|
│ ├─ RustBridge (stdin/stdout) │
|
||||||
│ └─ Config & S3 descriptor │
|
│ └─ Config & S3 descriptor │
|
||||||
└─────────────────────────────────┘
|
└─────────────────────────────────┘
|
||||||
@@ -326,9 +380,9 @@ smarts3 uses a **hybrid Rust + TypeScript** architecture:
|
|||||||
|
|
||||||
**Why Rust?** The TypeScript implementation had critical perf issues: OOM on multipart uploads (parts buffered in memory), double stream copying, file descriptor leaks on HEAD requests, full-file reads for range requests, and no backpressure. The Rust binary solves all of these with streaming I/O, zero-copy, and direct `seek()` for range requests.
|
**Why Rust?** The TypeScript implementation had critical perf issues: OOM on multipart uploads (parts buffered in memory), double stream copying, file descriptor leaks on HEAD requests, full-file reads for range requests, and no backpressure. The Rust binary solves all of these with streaming I/O, zero-copy, and direct `seek()` for range requests.
|
||||||
|
|
||||||
**IPC Protocol:** TypeScript spawns the `rusts3` binary with `--management` and communicates via newline-delimited JSON over stdin/stdout. Commands: `start`, `stop`, `createBucket`.
|
**IPC Protocol:** TypeScript spawns the `ruststorage` binary with `--management` and communicates via newline-delimited JSON over stdin/stdout. Commands: `start`, `stop`, `createBucket`.
|
||||||
|
|
||||||
### S3 Operations Supported
|
### S3-Compatible Operations Supported
|
||||||
|
|
||||||
| Operation | Method | Path |
|
| Operation | Method | Path |
|
||||||
|-----------|--------|------|
|
|-----------|--------|------|
|
||||||
@@ -347,27 +401,32 @@ smarts3 uses a **hybrid Rust + TypeScript** architecture:
|
|||||||
| CompleteMultipartUpload | `POST /{bucket}/{key}?uploadId` | |
|
| CompleteMultipartUpload | `POST /{bucket}/{key}?uploadId` | |
|
||||||
| AbortMultipartUpload | `DELETE /{bucket}/{key}?uploadId` | |
|
| AbortMultipartUpload | `DELETE /{bucket}/{key}?uploadId` | |
|
||||||
| ListMultipartUploads | `GET /{bucket}?uploads` | |
|
| ListMultipartUploads | `GET /{bucket}?uploads` | |
|
||||||
|
| GetBucketPolicy | `GET /{bucket}?policy` | |
|
||||||
|
| PutBucketPolicy | `PUT /{bucket}?policy` | |
|
||||||
|
| DeleteBucketPolicy | `DELETE /{bucket}?policy` | |
|
||||||
|
|
||||||
### On-Disk Format
|
### On-Disk Format
|
||||||
|
|
||||||
```
|
```
|
||||||
{storage.directory}/
|
{storage.directory}/
|
||||||
{bucket}/
|
{bucket}/
|
||||||
{key}._S3_object # Object data
|
{key}._storage_object # Object data
|
||||||
{key}._S3_object.metadata.json # Metadata (content-type, x-amz-meta-*, etc.)
|
{key}._storage_object.metadata.json # Metadata (content-type, x-amz-meta-*, etc.)
|
||||||
{key}._S3_object.md5 # Cached MD5 hash
|
{key}._storage_object.md5 # Cached MD5 hash
|
||||||
.multipart/
|
.multipart/
|
||||||
{upload-id}/
|
{upload-id}/
|
||||||
metadata.json # Upload metadata (bucket, key, parts)
|
metadata.json # Upload metadata (bucket, key, parts)
|
||||||
part-1 # Part data files
|
part-1 # Part data files
|
||||||
part-2
|
part-2
|
||||||
...
|
...
|
||||||
|
.policies/
|
||||||
|
{bucket}.policy.json # Bucket policy (IAM JSON format)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 🔗 Related Packages
|
## Related Packages
|
||||||
|
|
||||||
- [`@push.rocks/smartbucket`](https://code.foss.global/push.rocks/smartbucket) — High-level S3 abstraction layer
|
- [`@push.rocks/smartbucket`](https://code.foss.global/push.rocks/smartbucket) — High-level S3-compatible abstraction layer
|
||||||
- [`@push.rocks/smartrust`](https://code.foss.global/push.rocks/smartrust) — TypeScript ↔ Rust IPC bridge
|
- [`@push.rocks/smartrust`](https://code.foss.global/push.rocks/smartrust) — TypeScript <-> Rust IPC bridge
|
||||||
- [`@git.zone/tsrust`](https://code.foss.global/git.zone/tsrust) — Rust cross-compilation for npm packages
|
- [`@git.zone/tsrust`](https://code.foss.global/git.zone/tsrust) — Rust cross-compilation for npm packages
|
||||||
|
|
||||||
## License and Legal Information
|
## License and Legal Information
|
||||||
|
|||||||
47
rust/Cargo.lock
generated
47
rust/Cargo.lock
generated
@@ -197,6 +197,15 @@ version = "0.8.7"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cpufeatures"
|
||||||
|
version = "0.2.17"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.7"
|
version = "0.1.7"
|
||||||
@@ -215,6 +224,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer",
|
"block-buffer",
|
||||||
"crypto-common",
|
"crypto-common",
|
||||||
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -302,6 +312,21 @@ version = "0.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hex"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hmac"
|
||||||
|
version = "0.12.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||||
|
dependencies = [
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "http"
|
name = "http"
|
||||||
version = "1.4.0"
|
version = "1.4.0"
|
||||||
@@ -740,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
|
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rusts3"
|
name = "ruststorage"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
@@ -748,6 +773,8 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"clap",
|
"clap",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
|
"hex",
|
||||||
|
"hmac",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
"hyper",
|
"hyper",
|
||||||
"hyper-util",
|
"hyper-util",
|
||||||
@@ -756,6 +783,7 @@ dependencies = [
|
|||||||
"quick-xml",
|
"quick-xml",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sha2",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
@@ -820,6 +848,17 @@ dependencies = [
|
|||||||
"zmij",
|
"zmij",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sha2"
|
||||||
|
version = "0.10.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"cpufeatures",
|
||||||
|
"digest",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sharded-slab"
|
name = "sharded-slab"
|
||||||
version = "0.1.7"
|
version = "0.1.7"
|
||||||
@@ -873,6 +912,12 @@ version = "0.11.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "subtle"
|
||||||
|
version = "2.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "2.0.115"
|
version = "2.0.115"
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "rusts3"
|
name = "ruststorage"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "rusts3"
|
name = "ruststorage"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -28,3 +28,6 @@ percent-encoding = "2"
|
|||||||
url = "2"
|
url = "2"
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
futures-core = "0.3"
|
futures-core = "0.3"
|
||||||
|
hmac = "0.12"
|
||||||
|
sha2 = "0.10"
|
||||||
|
hex = "0.4"
|
||||||
|
|||||||
172
rust/src/action.rs
Normal file
172
rust/src/action.rs
Normal file
@@ -0,0 +1,172 @@
|
|||||||
|
use hyper::body::Incoming;
|
||||||
|
use hyper::{Method, Request};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
/// Storage actions that map to IAM permission strings.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum StorageAction {
|
||||||
|
ListAllMyBuckets,
|
||||||
|
CreateBucket,
|
||||||
|
DeleteBucket,
|
||||||
|
HeadBucket,
|
||||||
|
ListBucket,
|
||||||
|
GetObject,
|
||||||
|
HeadObject,
|
||||||
|
PutObject,
|
||||||
|
DeleteObject,
|
||||||
|
CopyObject,
|
||||||
|
ListBucketMultipartUploads,
|
||||||
|
AbortMultipartUpload,
|
||||||
|
InitiateMultipartUpload,
|
||||||
|
UploadPart,
|
||||||
|
CompleteMultipartUpload,
|
||||||
|
GetBucketPolicy,
|
||||||
|
PutBucketPolicy,
|
||||||
|
DeleteBucketPolicy,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StorageAction {
|
||||||
|
/// Return the IAM-style action string (e.g. "s3:GetObject").
|
||||||
|
pub fn iam_action(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
StorageAction::ListAllMyBuckets => "s3:ListAllMyBuckets",
|
||||||
|
StorageAction::CreateBucket => "s3:CreateBucket",
|
||||||
|
StorageAction::DeleteBucket => "s3:DeleteBucket",
|
||||||
|
StorageAction::HeadBucket => "s3:ListBucket",
|
||||||
|
StorageAction::ListBucket => "s3:ListBucket",
|
||||||
|
StorageAction::GetObject => "s3:GetObject",
|
||||||
|
StorageAction::HeadObject => "s3:GetObject",
|
||||||
|
StorageAction::PutObject => "s3:PutObject",
|
||||||
|
StorageAction::DeleteObject => "s3:DeleteObject",
|
||||||
|
StorageAction::CopyObject => "s3:PutObject",
|
||||||
|
StorageAction::ListBucketMultipartUploads => "s3:ListBucketMultipartUploads",
|
||||||
|
StorageAction::AbortMultipartUpload => "s3:AbortMultipartUpload",
|
||||||
|
StorageAction::InitiateMultipartUpload => "s3:PutObject",
|
||||||
|
StorageAction::UploadPart => "s3:PutObject",
|
||||||
|
StorageAction::CompleteMultipartUpload => "s3:PutObject",
|
||||||
|
StorageAction::GetBucketPolicy => "s3:GetBucketPolicy",
|
||||||
|
StorageAction::PutBucketPolicy => "s3:PutBucketPolicy",
|
||||||
|
StorageAction::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Context extracted from a request, used for policy evaluation.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct RequestContext {
|
||||||
|
pub action: StorageAction,
|
||||||
|
pub bucket: Option<String>,
|
||||||
|
pub key: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RequestContext {
|
||||||
|
/// Build the ARN for this request's resource.
|
||||||
|
pub fn resource_arn(&self) -> String {
|
||||||
|
match (&self.bucket, &self.key) {
|
||||||
|
(Some(bucket), Some(key)) => format!("arn:aws:s3:::{}/{}", bucket, key),
|
||||||
|
(Some(bucket), None) => format!("arn:aws:s3:::{}", bucket),
|
||||||
|
_ => "arn:aws:s3:::*".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the storage action from an incoming HTTP request.
|
||||||
|
pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
|
||||||
|
let method = req.method().clone();
|
||||||
|
let path = req.uri().path().to_string();
|
||||||
|
let query_string = req.uri().query().unwrap_or("").to_string();
|
||||||
|
let query = parse_query_simple(&query_string);
|
||||||
|
|
||||||
|
let segments: Vec<&str> = path
|
||||||
|
.trim_start_matches('/')
|
||||||
|
.splitn(2, '/')
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
match segments.len() {
|
||||||
|
0 => {
|
||||||
|
// Root: GET / -> ListBuckets
|
||||||
|
RequestContext {
|
||||||
|
action: StorageAction::ListAllMyBuckets,
|
||||||
|
bucket: None,
|
||||||
|
key: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
1 => {
|
||||||
|
let bucket = percent_decode(segments[0]);
|
||||||
|
let has_policy = query.contains_key("policy");
|
||||||
|
let has_uploads = query.contains_key("uploads");
|
||||||
|
|
||||||
|
let action = match (&method, has_policy, has_uploads) {
|
||||||
|
(&Method::GET, true, _) => StorageAction::GetBucketPolicy,
|
||||||
|
(&Method::PUT, true, _) => StorageAction::PutBucketPolicy,
|
||||||
|
(&Method::DELETE, true, _) => StorageAction::DeleteBucketPolicy,
|
||||||
|
(&Method::GET, _, true) => StorageAction::ListBucketMultipartUploads,
|
||||||
|
(&Method::GET, _, _) => StorageAction::ListBucket,
|
||||||
|
(&Method::PUT, _, _) => StorageAction::CreateBucket,
|
||||||
|
(&Method::DELETE, _, _) => StorageAction::DeleteBucket,
|
||||||
|
(&Method::HEAD, _, _) => StorageAction::HeadBucket,
|
||||||
|
_ => StorageAction::ListBucket,
|
||||||
|
};
|
||||||
|
|
||||||
|
RequestContext {
|
||||||
|
action,
|
||||||
|
bucket: Some(bucket),
|
||||||
|
key: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
2 => {
|
||||||
|
let bucket = percent_decode(segments[0]);
|
||||||
|
let key = percent_decode(segments[1]);
|
||||||
|
|
||||||
|
let has_copy_source = req.headers().contains_key("x-amz-copy-source");
|
||||||
|
let has_part_number = query.contains_key("partNumber");
|
||||||
|
let has_upload_id = query.contains_key("uploadId");
|
||||||
|
let has_uploads = query.contains_key("uploads");
|
||||||
|
|
||||||
|
let action = match &method {
|
||||||
|
&Method::PUT if has_part_number && has_upload_id => StorageAction::UploadPart,
|
||||||
|
&Method::PUT if has_copy_source => StorageAction::CopyObject,
|
||||||
|
&Method::PUT => StorageAction::PutObject,
|
||||||
|
&Method::GET => StorageAction::GetObject,
|
||||||
|
&Method::HEAD => StorageAction::HeadObject,
|
||||||
|
&Method::DELETE if has_upload_id => StorageAction::AbortMultipartUpload,
|
||||||
|
&Method::DELETE => StorageAction::DeleteObject,
|
||||||
|
&Method::POST if has_uploads => StorageAction::InitiateMultipartUpload,
|
||||||
|
&Method::POST if has_upload_id => StorageAction::CompleteMultipartUpload,
|
||||||
|
_ => StorageAction::GetObject,
|
||||||
|
};
|
||||||
|
|
||||||
|
RequestContext {
|
||||||
|
action,
|
||||||
|
bucket: Some(bucket),
|
||||||
|
key: Some(key),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => RequestContext {
|
||||||
|
action: StorageAction::ListAllMyBuckets,
|
||||||
|
bucket: None,
|
||||||
|
key: None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_query_simple(query_string: &str) -> HashMap<String, String> {
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
if query_string.is_empty() {
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
for pair in query_string.split('&') {
|
||||||
|
let mut parts = pair.splitn(2, '=');
|
||||||
|
let key = parts.next().unwrap_or("");
|
||||||
|
let value = parts.next().unwrap_or("");
|
||||||
|
map.insert(key.to_string(), value.to_string());
|
||||||
|
}
|
||||||
|
map
|
||||||
|
}
|
||||||
|
|
||||||
|
fn percent_decode(s: &str) -> String {
|
||||||
|
percent_encoding::percent_decode_str(s)
|
||||||
|
.decode_utf8_lossy()
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
310
rust/src/auth.rs
Normal file
310
rust/src/auth.rs
Normal file
@@ -0,0 +1,310 @@
|
|||||||
|
use hmac::{Hmac, Mac};
|
||||||
|
use hyper::body::Incoming;
|
||||||
|
use hyper::Request;
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use crate::config::{Credential, SmartStorageConfig};
|
||||||
|
use crate::error::StorageError;
|
||||||
|
|
||||||
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
|
|
||||||
|
/// The identity of an authenticated caller.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct AuthenticatedIdentity {
|
||||||
|
pub access_key_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parsed components of an AWS4-HMAC-SHA256 Authorization header.
|
||||||
|
struct SigV4Header {
|
||||||
|
access_key_id: String,
|
||||||
|
date_stamp: String,
|
||||||
|
region: String,
|
||||||
|
signed_headers: Vec<String>,
|
||||||
|
signature: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify the request's SigV4 signature. Returns the caller identity on success.
|
||||||
|
pub fn verify_request(
|
||||||
|
req: &Request<Incoming>,
|
||||||
|
config: &SmartStorageConfig,
|
||||||
|
) -> Result<AuthenticatedIdentity, StorageError> {
|
||||||
|
let auth_header = req
|
||||||
|
.headers()
|
||||||
|
.get("authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
|
||||||
|
// Reject SigV2
|
||||||
|
if auth_header.starts_with("AWS ") {
|
||||||
|
return Err(StorageError::authorization_header_malformed());
|
||||||
|
}
|
||||||
|
|
||||||
|
if !auth_header.starts_with("AWS4-HMAC-SHA256") {
|
||||||
|
return Err(StorageError::authorization_header_malformed());
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed = parse_auth_header(auth_header)?;
|
||||||
|
|
||||||
|
// Look up credential
|
||||||
|
let credential = find_credential(&parsed.access_key_id, config)
|
||||||
|
.ok_or_else(StorageError::invalid_access_key_id)?;
|
||||||
|
|
||||||
|
// Get x-amz-date
|
||||||
|
let amz_date = req
|
||||||
|
.headers()
|
||||||
|
.get("x-amz-date")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.or_else(|| {
|
||||||
|
req.headers()
|
||||||
|
.get("date")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
})
|
||||||
|
.ok_or_else(|| StorageError::missing_security_header("Missing x-amz-date header"))?;
|
||||||
|
|
||||||
|
// Enforce 15-min clock skew
|
||||||
|
check_clock_skew(amz_date)?;
|
||||||
|
|
||||||
|
// Get payload hash
|
||||||
|
let content_sha256 = req
|
||||||
|
.headers()
|
||||||
|
.get("x-amz-content-sha256")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("UNSIGNED-PAYLOAD");
|
||||||
|
|
||||||
|
// Build canonical request
|
||||||
|
let canonical_request = build_canonical_request(req, &parsed.signed_headers, content_sha256);
|
||||||
|
|
||||||
|
// Build string to sign
|
||||||
|
let scope = format!(
|
||||||
|
"{}/{}/s3/aws4_request",
|
||||||
|
parsed.date_stamp, parsed.region
|
||||||
|
);
|
||||||
|
let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes()));
|
||||||
|
let string_to_sign = format!(
|
||||||
|
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||||
|
amz_date, scope, canonical_hash
|
||||||
|
);
|
||||||
|
|
||||||
|
// Derive signing key
|
||||||
|
let signing_key = derive_signing_key(
|
||||||
|
&credential.secret_access_key,
|
||||||
|
&parsed.date_stamp,
|
||||||
|
&parsed.region,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Compute signature
|
||||||
|
let computed = hmac_sha256(&signing_key, string_to_sign.as_bytes());
|
||||||
|
let computed_hex = hex::encode(&computed);
|
||||||
|
|
||||||
|
// Constant-time comparison
|
||||||
|
if !constant_time_eq(computed_hex.as_bytes(), parsed.signature.as_bytes()) {
|
||||||
|
return Err(StorageError::signature_does_not_match());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(AuthenticatedIdentity {
|
||||||
|
access_key_id: parsed.access_key_id,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the Authorization header into its components.
|
||||||
|
fn parse_auth_header(header: &str) -> Result<SigV4Header, StorageError> {
|
||||||
|
// Format: AWS4-HMAC-SHA256 Credential=KEY/YYYYMMDD/region/s3/aws4_request, SignedHeaders=h1;h2, Signature=hex
|
||||||
|
let after_algo = header
|
||||||
|
.strip_prefix("AWS4-HMAC-SHA256")
|
||||||
|
.ok_or_else(StorageError::authorization_header_malformed)?
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
let mut credential_str = None;
|
||||||
|
let mut signed_headers_str = None;
|
||||||
|
let mut signature_str = None;
|
||||||
|
|
||||||
|
for part in after_algo.split(',') {
|
||||||
|
let part = part.trim();
|
||||||
|
if let Some(val) = part.strip_prefix("Credential=") {
|
||||||
|
credential_str = Some(val.trim());
|
||||||
|
} else if let Some(val) = part.strip_prefix("SignedHeaders=") {
|
||||||
|
signed_headers_str = Some(val.trim());
|
||||||
|
} else if let Some(val) = part.strip_prefix("Signature=") {
|
||||||
|
signature_str = Some(val.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let credential_str = credential_str
|
||||||
|
.ok_or_else(StorageError::authorization_header_malformed)?;
|
||||||
|
let signed_headers_str = signed_headers_str
|
||||||
|
.ok_or_else(StorageError::authorization_header_malformed)?;
|
||||||
|
let signature = signature_str
|
||||||
|
.ok_or_else(StorageError::authorization_header_malformed)?
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
// Parse credential: KEY/YYYYMMDD/region/s3/aws4_request
|
||||||
|
let cred_parts: Vec<&str> = credential_str.splitn(5, '/').collect();
|
||||||
|
if cred_parts.len() < 5 {
|
||||||
|
return Err(StorageError::authorization_header_malformed());
|
||||||
|
}
|
||||||
|
|
||||||
|
let access_key_id = cred_parts[0].to_string();
|
||||||
|
let date_stamp = cred_parts[1].to_string();
|
||||||
|
let region = cred_parts[2].to_string();
|
||||||
|
|
||||||
|
let signed_headers: Vec<String> = signed_headers_str
|
||||||
|
.split(';')
|
||||||
|
.map(|s| s.trim().to_lowercase())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(SigV4Header {
|
||||||
|
access_key_id,
|
||||||
|
date_stamp,
|
||||||
|
region,
|
||||||
|
signed_headers,
|
||||||
|
signature,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Find a credential by access key ID.
|
||||||
|
fn find_credential<'a>(access_key_id: &str, config: &'a SmartStorageConfig) -> Option<&'a Credential> {
|
||||||
|
config
|
||||||
|
.auth
|
||||||
|
.credentials
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.access_key_id == access_key_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check clock skew (15 minutes max).
|
||||||
|
fn check_clock_skew(amz_date: &str) -> Result<(), StorageError> {
|
||||||
|
// Parse ISO 8601 basic format: YYYYMMDDTHHMMSSZ
|
||||||
|
let parsed = chrono::NaiveDateTime::parse_from_str(amz_date, "%Y%m%dT%H%M%SZ")
|
||||||
|
.map_err(|_| StorageError::authorization_header_malformed())?;
|
||||||
|
|
||||||
|
let request_time = chrono::DateTime::<chrono::Utc>::from_naive_utc_and_offset(parsed, chrono::Utc);
|
||||||
|
let now = chrono::Utc::now();
|
||||||
|
let diff = (now - request_time).num_seconds().unsigned_abs();
|
||||||
|
|
||||||
|
if diff > 15 * 60 {
|
||||||
|
return Err(StorageError::request_time_too_skewed());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the canonical request string.
|
||||||
|
fn build_canonical_request(
|
||||||
|
req: &Request<Incoming>,
|
||||||
|
signed_headers: &[String],
|
||||||
|
payload_hash: &str,
|
||||||
|
) -> String {
|
||||||
|
let method = req.method().as_str();
|
||||||
|
let uri_path = req.uri().path();
|
||||||
|
|
||||||
|
// Canonical URI: the path, already percent-encoded by the client
|
||||||
|
let canonical_uri = if uri_path.is_empty() { "/" } else { uri_path };
|
||||||
|
|
||||||
|
// Canonical query string: sorted key=value pairs
|
||||||
|
let canonical_query = build_canonical_query(req.uri().query().unwrap_or(""));
|
||||||
|
|
||||||
|
// Canonical headers: sorted by lowercase header name
|
||||||
|
let canonical_headers = build_canonical_headers(req, signed_headers);
|
||||||
|
|
||||||
|
// Signed headers string
|
||||||
|
let signed_headers_str = signed_headers.join(";");
|
||||||
|
|
||||||
|
// Payload hash — accept UNSIGNED-PAYLOAD and STREAMING-AWS4-HMAC-SHA256-PAYLOAD as-is
|
||||||
|
let effective_payload_hash = if payload_hash == "UNSIGNED-PAYLOAD"
|
||||||
|
|| payload_hash == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||||
|
{
|
||||||
|
payload_hash.to_string()
|
||||||
|
} else {
|
||||||
|
payload_hash.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
format!(
|
||||||
|
"{}\n{}\n{}\n{}\n{}\n{}",
|
||||||
|
method,
|
||||||
|
canonical_uri,
|
||||||
|
canonical_query,
|
||||||
|
canonical_headers,
|
||||||
|
signed_headers_str,
|
||||||
|
effective_payload_hash
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build canonical query string (sorted key=value pairs).
|
||||||
|
fn build_canonical_query(query: &str) -> String {
|
||||||
|
if query.is_empty() {
|
||||||
|
return String::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut pairs: Vec<(String, String)> = Vec::new();
|
||||||
|
for pair in query.split('&') {
|
||||||
|
let mut parts = pair.splitn(2, '=');
|
||||||
|
let key = parts.next().unwrap_or("");
|
||||||
|
let value = parts.next().unwrap_or("");
|
||||||
|
pairs.push((key.to_string(), value.to_string()));
|
||||||
|
}
|
||||||
|
pairs.sort();
|
||||||
|
|
||||||
|
pairs
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| format!("{}={}", k, v))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("&")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build canonical headers string.
|
||||||
|
fn build_canonical_headers(req: &Request<Incoming>, signed_headers: &[String]) -> String {
|
||||||
|
let mut header_map: HashMap<String, Vec<String>> = HashMap::new();
|
||||||
|
|
||||||
|
for (name, value) in req.headers() {
|
||||||
|
let name_lower = name.as_str().to_lowercase();
|
||||||
|
if signed_headers.contains(&name_lower) {
|
||||||
|
if let Ok(val) = value.to_str() {
|
||||||
|
header_map
|
||||||
|
.entry(name_lower)
|
||||||
|
.or_default()
|
||||||
|
.push(val.trim().to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut result = String::new();
|
||||||
|
for header_name in signed_headers {
|
||||||
|
let values = header_map
|
||||||
|
.get(header_name)
|
||||||
|
.map(|v| v.join(","))
|
||||||
|
.unwrap_or_default();
|
||||||
|
result.push_str(header_name);
|
||||||
|
result.push(':');
|
||||||
|
result.push_str(&values);
|
||||||
|
result.push('\n');
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Derive the signing key via 4-step HMAC chain.
|
||||||
|
fn derive_signing_key(secret_key: &str, date_stamp: &str, region: &str) -> Vec<u8> {
|
||||||
|
let k_secret = format!("AWS4{}", secret_key);
|
||||||
|
let k_date = hmac_sha256(k_secret.as_bytes(), date_stamp.as_bytes());
|
||||||
|
let k_region = hmac_sha256(&k_date, region.as_bytes());
|
||||||
|
let k_service = hmac_sha256(&k_region, b"s3");
|
||||||
|
hmac_sha256(&k_service, b"aws4_request")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute HMAC-SHA256.
|
||||||
|
fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec<u8> {
|
||||||
|
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
|
||||||
|
mac.update(data);
|
||||||
|
mac.finalize().into_bytes().to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Constant-time byte comparison.
|
||||||
|
fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
|
||||||
|
if a.len() != b.len() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let mut diff = 0u8;
|
||||||
|
for (x, y) in a.iter().zip(b.iter()) {
|
||||||
|
diff |= x ^ y;
|
||||||
|
}
|
||||||
|
diff == 0
|
||||||
|
}
|
||||||
@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct S3Config {
|
pub struct SmartStorageConfig {
|
||||||
pub server: ServerConfig,
|
pub server: ServerConfig,
|
||||||
pub storage: StorageConfig,
|
pub storage: StorageConfig,
|
||||||
pub auth: AuthConfig,
|
pub auth: AuthConfig,
|
||||||
@@ -18,6 +18,12 @@ pub struct ServerConfig {
|
|||||||
pub port: u16,
|
pub port: u16,
|
||||||
pub address: String,
|
pub address: String,
|
||||||
pub silent: bool,
|
pub silent: bool,
|
||||||
|
#[serde(default = "default_region")]
|
||||||
|
pub region: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_region() -> String {
|
||||||
|
"us-east-1".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
|||||||
@@ -1,16 +1,14 @@
|
|||||||
use hyper::{Response, StatusCode};
|
use hyper::StatusCode;
|
||||||
use http_body_util::Full;
|
|
||||||
use bytes::Bytes;
|
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
#[error("S3Error({code}): {message}")]
|
#[error("StorageError({code}): {message}")]
|
||||||
pub struct S3Error {
|
pub struct StorageError {
|
||||||
pub code: String,
|
pub code: String,
|
||||||
pub message: String,
|
pub message: String,
|
||||||
pub status: StatusCode,
|
pub status: StatusCode,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl S3Error {
|
impl StorageError {
|
||||||
pub fn new(code: &str, message: &str, status: StatusCode) -> Self {
|
pub fn new(code: &str, message: &str, status: StatusCode) -> Self {
|
||||||
Self {
|
Self {
|
||||||
code: code.to_string(),
|
code: code.to_string(),
|
||||||
@@ -51,20 +49,58 @@ impl S3Error {
|
|||||||
Self::new("InvalidRequest", msg, StatusCode::BAD_REQUEST)
|
Self::new("InvalidRequest", msg, StatusCode::BAD_REQUEST)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn signature_does_not_match() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"SignatureDoesNotMatch",
|
||||||
|
"The request signature we calculated does not match the signature you provided.",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn invalid_access_key_id() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"InvalidAccessKeyId",
|
||||||
|
"The AWS Access Key Id you provided does not exist in our records.",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn request_time_too_skewed() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"RequestTimeTooSkewed",
|
||||||
|
"The difference between the request time and the current time is too large.",
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn authorization_header_malformed() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"AuthorizationHeaderMalformed",
|
||||||
|
"The authorization header is malformed.",
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn missing_security_header(msg: &str) -> Self {
|
||||||
|
Self::new("MissingSecurityHeader", msg, StatusCode::BAD_REQUEST)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn no_such_bucket_policy() -> Self {
|
||||||
|
Self::new(
|
||||||
|
"NoSuchBucketPolicy",
|
||||||
|
"The bucket policy does not exist.",
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn malformed_policy(msg: &str) -> Self {
|
||||||
|
Self::new("MalformedPolicy", msg, StatusCode::BAD_REQUEST)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn to_xml(&self) -> String {
|
pub fn to_xml(&self) -> String {
|
||||||
format!(
|
format!(
|
||||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>{}</Code><Message>{}</Message></Error>",
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>{}</Code><Message>{}</Message></Error>",
|
||||||
self.code, self.message
|
self.code, self.message
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn to_response(&self, request_id: &str) -> Response<Full<Bytes>> {
|
|
||||||
let xml = self.to_xml();
|
|
||||||
Response::builder()
|
|
||||||
.status(self.status)
|
|
||||||
.header("content-type", "application/xml")
|
|
||||||
.header("x-amz-request-id", request_id)
|
|
||||||
.body(Full::new(Bytes::from(xml)))
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
|
mod action;
|
||||||
|
mod auth;
|
||||||
mod config;
|
mod config;
|
||||||
mod management;
|
mod management;
|
||||||
mod s3_error;
|
mod policy;
|
||||||
|
mod error;
|
||||||
mod server;
|
mod server;
|
||||||
mod storage;
|
mod storage;
|
||||||
mod xml_response;
|
mod xml_response;
|
||||||
@@ -8,7 +11,7 @@ mod xml_response;
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
|
||||||
#[derive(Parser)]
|
#[derive(Parser)]
|
||||||
#[command(name = "rusts3", about = "High-performance S3-compatible server")]
|
#[command(name = "ruststorage", about = "High-performance S3-compatible storage server")]
|
||||||
struct Cli {
|
struct Cli {
|
||||||
/// Run in management mode (IPC via stdin/stdout)
|
/// Run in management mode (IPC via stdin/stdout)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
@@ -35,7 +38,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
management::management_loop().await?;
|
management::management_loop().await?;
|
||||||
} else {
|
} else {
|
||||||
eprintln!("rusts3: use --management flag for IPC mode");
|
eprintln!("ruststorage: use --management flag for IPC mode");
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ use serde_json::Value;
|
|||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
|
|
||||||
use crate::config::S3Config;
|
use crate::config::SmartStorageConfig;
|
||||||
use crate::server::S3Server;
|
use crate::server::StorageServer;
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct IpcRequest {
|
struct IpcRequest {
|
||||||
@@ -62,7 +62,7 @@ pub async fn management_loop() -> Result<()> {
|
|||||||
data: serde_json::json!({}),
|
data: serde_json::json!({}),
|
||||||
});
|
});
|
||||||
|
|
||||||
let mut server: Option<S3Server> = None;
|
let mut server: Option<StorageServer> = None;
|
||||||
let stdin = BufReader::new(tokio::io::stdin());
|
let stdin = BufReader::new(tokio::io::stdin());
|
||||||
let mut lines = stdin.lines();
|
let mut lines = stdin.lines();
|
||||||
|
|
||||||
@@ -87,11 +87,11 @@ pub async fn management_loop() -> Result<()> {
|
|||||||
"start" => {
|
"start" => {
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct StartParams {
|
struct StartParams {
|
||||||
config: S3Config,
|
config: SmartStorageConfig,
|
||||||
}
|
}
|
||||||
match serde_json::from_value::<StartParams>(req.params) {
|
match serde_json::from_value::<StartParams>(req.params) {
|
||||||
Ok(params) => {
|
Ok(params) => {
|
||||||
match S3Server::start(params.config).await {
|
match StorageServer::start(params.config).await {
|
||||||
Ok(s) => {
|
Ok(s) => {
|
||||||
server = Some(s);
|
server = Some(s);
|
||||||
send_response(id, serde_json::json!({}));
|
send_response(id, serde_json::json!({}));
|
||||||
|
|||||||
429
rust/src/policy.rs
Normal file
429
rust/src/policy.rs
Normal file
@@ -0,0 +1,429 @@
|
|||||||
|
use serde::{Deserialize, Deserializer, Serialize};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use tokio::fs;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
use crate::action::RequestContext;
|
||||||
|
use crate::auth::AuthenticatedIdentity;
|
||||||
|
use crate::error::StorageError;
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Policy data model
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct BucketPolicy {
|
||||||
|
#[serde(rename = "Version")]
|
||||||
|
pub version: String,
|
||||||
|
#[serde(rename = "Statement")]
|
||||||
|
pub statements: Vec<PolicyStatement>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct PolicyStatement {
|
||||||
|
#[serde(rename = "Sid", default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub sid: Option<String>,
|
||||||
|
#[serde(rename = "Effect")]
|
||||||
|
pub effect: PolicyEffect,
|
||||||
|
#[serde(rename = "Principal", deserialize_with = "deserialize_principal")]
|
||||||
|
pub principal: Principal,
|
||||||
|
#[serde(rename = "Action", deserialize_with = "deserialize_string_or_vec")]
|
||||||
|
pub action: Vec<String>,
|
||||||
|
#[serde(rename = "Resource", deserialize_with = "deserialize_string_or_vec")]
|
||||||
|
pub resource: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||||
|
pub enum PolicyEffect {
|
||||||
|
Allow,
|
||||||
|
Deny,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum Principal {
|
||||||
|
Wildcard,
|
||||||
|
Aws(Vec<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for Principal {
|
||||||
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: serde::Serializer,
|
||||||
|
{
|
||||||
|
match self {
|
||||||
|
Principal::Wildcard => serializer.serialize_str("*"),
|
||||||
|
Principal::Aws(ids) => {
|
||||||
|
use serde::ser::SerializeMap;
|
||||||
|
let mut map = serializer.serialize_map(Some(1))?;
|
||||||
|
if ids.len() == 1 {
|
||||||
|
map.serialize_entry("AWS", &ids[0])?;
|
||||||
|
} else {
|
||||||
|
map.serialize_entry("AWS", ids)?;
|
||||||
|
}
|
||||||
|
map.end()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deserialize_principal<'de, D>(deserializer: D) -> Result<Principal, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum PrincipalRaw {
|
||||||
|
Star(String),
|
||||||
|
Map(HashMap<String, StringOrVec>),
|
||||||
|
}
|
||||||
|
|
||||||
|
let raw = PrincipalRaw::deserialize(deserializer)?;
|
||||||
|
match raw {
|
||||||
|
PrincipalRaw::Star(s) if s == "*" => Ok(Principal::Wildcard),
|
||||||
|
PrincipalRaw::Star(_) => Err(serde::de::Error::custom(
|
||||||
|
"Principal string must be \"*\"",
|
||||||
|
)),
|
||||||
|
PrincipalRaw::Map(map) => {
|
||||||
|
if let Some(aws) = map.get("AWS") {
|
||||||
|
Ok(Principal::Aws(aws.clone().into_vec()))
|
||||||
|
} else {
|
||||||
|
Err(serde::de::Error::custom("Principal map must contain \"AWS\" key"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum StringOrVec {
|
||||||
|
Single(String),
|
||||||
|
Multiple(Vec<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StringOrVec {
|
||||||
|
fn into_vec(self) -> Vec<String> {
|
||||||
|
match self {
|
||||||
|
StringOrVec::Single(s) => vec![s],
|
||||||
|
StringOrVec::Multiple(v) => v,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deserialize_string_or_vec<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
|
||||||
|
where
|
||||||
|
D: Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let raw = StringOrVec::deserialize(deserializer)?;
|
||||||
|
Ok(raw.into_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Policy evaluation
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum PolicyDecision {
|
||||||
|
Allow,
|
||||||
|
Deny,
|
||||||
|
NoOpinion,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evaluate a bucket policy against a request context and caller identity.
|
||||||
|
pub fn evaluate_policy(
|
||||||
|
policy: &BucketPolicy,
|
||||||
|
ctx: &RequestContext,
|
||||||
|
identity: Option<&AuthenticatedIdentity>,
|
||||||
|
) -> PolicyDecision {
|
||||||
|
let resource_arn = ctx.resource_arn();
|
||||||
|
let iam_action = ctx.action.iam_action();
|
||||||
|
let mut has_allow = false;
|
||||||
|
|
||||||
|
for stmt in &policy.statements {
|
||||||
|
// Check principal match
|
||||||
|
if !principal_matches(&stmt.principal, identity) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check action match
|
||||||
|
if !action_matches(&stmt.action, iam_action) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check resource match
|
||||||
|
if !resource_matches(&stmt.resource, &resource_arn, ctx.bucket.as_deref()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Statement matches — apply effect
|
||||||
|
match stmt.effect {
|
||||||
|
PolicyEffect::Deny => return PolicyDecision::Deny,
|
||||||
|
PolicyEffect::Allow => has_allow = true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if has_allow {
|
||||||
|
PolicyDecision::Allow
|
||||||
|
} else {
|
||||||
|
PolicyDecision::NoOpinion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if the principal matches the caller.
|
||||||
|
fn principal_matches(principal: &Principal, identity: Option<&AuthenticatedIdentity>) -> bool {
|
||||||
|
match principal {
|
||||||
|
Principal::Wildcard => true,
|
||||||
|
Principal::Aws(ids) => {
|
||||||
|
if let Some(id) = identity {
|
||||||
|
ids.iter().any(|arn| {
|
||||||
|
// Match against full ARN or just the access key ID
|
||||||
|
arn == "*" || arn.ends_with(&id.access_key_id)
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if the action matches. Supports wildcard `s3:*` and `*`.
|
||||||
|
fn action_matches(policy_actions: &[String], request_action: &str) -> bool {
|
||||||
|
for pa in policy_actions {
|
||||||
|
if pa == "*" || pa == "s3:*" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if pa.eq_ignore_ascii_case(request_action) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Simple prefix wildcard: "s3:Get*" matches "s3:GetObject"
|
||||||
|
if let Some(prefix) = pa.strip_suffix('*') {
|
||||||
|
if request_action
|
||||||
|
.to_lowercase()
|
||||||
|
.starts_with(&prefix.to_lowercase())
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if the resource matches. Supports wildcard patterns.
|
||||||
|
fn resource_matches(policy_resources: &[String], request_arn: &str, bucket: Option<&str>) -> bool {
|
||||||
|
for pr in policy_resources {
|
||||||
|
if pr == "*" {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if arn_pattern_matches(pr, request_arn) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Also check bucket-level ARN if the request is for an object
|
||||||
|
if let Some(b) = bucket {
|
||||||
|
let bucket_arn = format!("arn:aws:s3:::{}", b);
|
||||||
|
if arn_pattern_matches(pr, &bucket_arn) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Simple ARN pattern matching with `*` and `?` wildcards.
|
||||||
|
fn arn_pattern_matches(pattern: &str, value: &str) -> bool {
|
||||||
|
// Handle trailing /* specifically: arn:aws:s3:::bucket/* matches arn:aws:s3:::bucket/anything
|
||||||
|
if pattern.ends_with("/*") {
|
||||||
|
let prefix = &pattern[..pattern.len() - 1]; // Remove trailing *
|
||||||
|
if value.starts_with(prefix) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Also match exact bucket without trailing /
|
||||||
|
let bucket_only = &pattern[..pattern.len() - 2];
|
||||||
|
if value == bucket_only {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
simple_wildcard_match(pattern, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn simple_wildcard_match(pattern: &str, value: &str) -> bool {
|
||||||
|
let pat_bytes = pattern.as_bytes();
|
||||||
|
let val_bytes = value.as_bytes();
|
||||||
|
let mut pi = 0;
|
||||||
|
let mut vi = 0;
|
||||||
|
let mut star_pi = usize::MAX;
|
||||||
|
let mut star_vi = 0;
|
||||||
|
|
||||||
|
while vi < val_bytes.len() {
|
||||||
|
if pi < pat_bytes.len() && (pat_bytes[pi] == b'?' || pat_bytes[pi] == val_bytes[vi]) {
|
||||||
|
pi += 1;
|
||||||
|
vi += 1;
|
||||||
|
} else if pi < pat_bytes.len() && pat_bytes[pi] == b'*' {
|
||||||
|
star_pi = pi;
|
||||||
|
star_vi = vi;
|
||||||
|
pi += 1;
|
||||||
|
} else if star_pi != usize::MAX {
|
||||||
|
pi = star_pi + 1;
|
||||||
|
star_vi += 1;
|
||||||
|
vi = star_vi;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while pi < pat_bytes.len() && pat_bytes[pi] == b'*' {
|
||||||
|
pi += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
pi == pat_bytes.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Policy validation
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
const MAX_POLICY_SIZE: usize = 20 * 1024; // 20 KB
|
||||||
|
|
||||||
|
pub fn validate_policy(json: &str) -> Result<BucketPolicy, StorageError> {
|
||||||
|
if json.len() > MAX_POLICY_SIZE {
|
||||||
|
return Err(StorageError::malformed_policy("Policy exceeds maximum size of 20KB"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy: BucketPolicy =
|
||||||
|
serde_json::from_str(json).map_err(|e| StorageError::malformed_policy(&e.to_string()))?;
|
||||||
|
|
||||||
|
if policy.version != "2012-10-17" {
|
||||||
|
return Err(StorageError::malformed_policy(
|
||||||
|
"Policy version must be \"2012-10-17\"",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if policy.statements.is_empty() {
|
||||||
|
return Err(StorageError::malformed_policy(
|
||||||
|
"Policy must contain at least one statement",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (i, stmt) in policy.statements.iter().enumerate() {
|
||||||
|
if stmt.action.is_empty() {
|
||||||
|
return Err(StorageError::malformed_policy(&format!(
|
||||||
|
"Statement {} has no actions",
|
||||||
|
i
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
for action in &stmt.action {
|
||||||
|
if action != "*" && !action.starts_with("s3:") {
|
||||||
|
return Err(StorageError::malformed_policy(&format!(
|
||||||
|
"Action \"{}\" must start with \"s3:\"",
|
||||||
|
action
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stmt.resource.is_empty() {
|
||||||
|
return Err(StorageError::malformed_policy(&format!(
|
||||||
|
"Statement {} has no resources",
|
||||||
|
i
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
for resource in &stmt.resource {
|
||||||
|
if resource != "*" && !resource.starts_with("arn:aws:s3:::") {
|
||||||
|
return Err(StorageError::malformed_policy(&format!(
|
||||||
|
"Resource \"{}\" must start with \"arn:aws:s3:::\"",
|
||||||
|
resource
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(policy)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// PolicyStore — in-memory cache + disk
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
pub struct PolicyStore {
|
||||||
|
policies: RwLock<HashMap<String, BucketPolicy>>,
|
||||||
|
policies_dir: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PolicyStore {
|
||||||
|
pub fn new(policies_dir: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
policies: RwLock::new(HashMap::new()),
|
||||||
|
policies_dir,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load all policies from disk into cache.
|
||||||
|
pub async fn load_from_disk(&self) -> anyhow::Result<()> {
|
||||||
|
let dir = &self.policies_dir;
|
||||||
|
if !dir.exists() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries = fs::read_dir(dir).await?;
|
||||||
|
let mut policies = HashMap::new();
|
||||||
|
|
||||||
|
while let Some(entry) = entries.next_entry().await? {
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
if let Some(bucket) = name.strip_suffix(".policy.json") {
|
||||||
|
match fs::read_to_string(entry.path()).await {
|
||||||
|
Ok(json) => match serde_json::from_str::<BucketPolicy>(&json) {
|
||||||
|
Ok(policy) => {
|
||||||
|
tracing::info!("Loaded policy for bucket: {}", bucket);
|
||||||
|
policies.insert(bucket.to_string(), policy);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("Failed to parse policy for {}: {}", bucket, e);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("Failed to read policy file {}: {}", name, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut cache = self.policies.write().await;
|
||||||
|
*cache = policies;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a policy for a bucket.
|
||||||
|
pub async fn get_policy(&self, bucket: &str) -> Option<BucketPolicy> {
|
||||||
|
let cache = self.policies.read().await;
|
||||||
|
cache.get(bucket).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Store a policy for a bucket (atomic write + cache update).
|
||||||
|
pub async fn put_policy(&self, bucket: &str, policy: BucketPolicy) -> anyhow::Result<()> {
|
||||||
|
let json = serde_json::to_string_pretty(&policy)?;
|
||||||
|
|
||||||
|
// Atomic write: temp file + rename
|
||||||
|
let policy_path = self.policies_dir.join(format!("{}.policy.json", bucket));
|
||||||
|
let temp_path = self
|
||||||
|
.policies_dir
|
||||||
|
.join(format!("{}.policy.json.tmp", bucket));
|
||||||
|
|
||||||
|
fs::write(&temp_path, &json).await?;
|
||||||
|
fs::rename(&temp_path, &policy_path).await?;
|
||||||
|
|
||||||
|
// Update cache
|
||||||
|
let mut cache = self.policies.write().await;
|
||||||
|
cache.insert(bucket.to_string(), policy);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete a policy for a bucket.
|
||||||
|
pub async fn delete_policy(&self, bucket: &str) -> anyhow::Result<()> {
|
||||||
|
let policy_path = self.policies_dir.join(format!("{}.policy.json", bucket));
|
||||||
|
let _ = fs::remove_file(&policy_path).await;
|
||||||
|
|
||||||
|
let mut cache = self.policies.write().await;
|
||||||
|
cache.remove(bucket);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,20 +18,22 @@ use tokio::sync::watch;
|
|||||||
use tokio_util::io::ReaderStream;
|
use tokio_util::io::ReaderStream;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::config::S3Config;
|
use crate::action::{self, RequestContext, StorageAction};
|
||||||
use crate::s3_error::S3Error;
|
use crate::auth::{self, AuthenticatedIdentity};
|
||||||
|
use crate::config::SmartStorageConfig;
|
||||||
|
use crate::policy::{self, PolicyDecision, PolicyStore};
|
||||||
|
use crate::error::StorageError;
|
||||||
use crate::storage::FileStore;
|
use crate::storage::FileStore;
|
||||||
use crate::xml_response;
|
use crate::xml_response;
|
||||||
|
|
||||||
pub struct S3Server {
|
pub struct StorageServer {
|
||||||
store: Arc<FileStore>,
|
store: Arc<FileStore>,
|
||||||
config: S3Config,
|
|
||||||
shutdown_tx: watch::Sender<bool>,
|
shutdown_tx: watch::Sender<bool>,
|
||||||
server_handle: tokio::task::JoinHandle<()>,
|
server_handle: tokio::task::JoinHandle<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl S3Server {
|
impl StorageServer {
|
||||||
pub async fn start(config: S3Config) -> Result<Self> {
|
pub async fn start(config: SmartStorageConfig) -> Result<Self> {
|
||||||
let store = Arc::new(FileStore::new(config.storage.directory.clone().into()));
|
let store = Arc::new(FileStore::new(config.storage.directory.clone().into()));
|
||||||
|
|
||||||
// Initialize or reset storage
|
// Initialize or reset storage
|
||||||
@@ -41,6 +43,10 @@ impl S3Server {
|
|||||||
store.initialize().await?;
|
store.initialize().await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initialize policy store
|
||||||
|
let policy_store = Arc::new(PolicyStore::new(store.policies_dir()));
|
||||||
|
policy_store.load_from_disk().await?;
|
||||||
|
|
||||||
let addr: SocketAddr = format!("{}:{}", config.address(), config.server.port)
|
let addr: SocketAddr = format!("{}:{}", config.address(), config.server.port)
|
||||||
.parse()?;
|
.parse()?;
|
||||||
|
|
||||||
@@ -49,6 +55,7 @@ impl S3Server {
|
|||||||
|
|
||||||
let server_store = store.clone();
|
let server_store = store.clone();
|
||||||
let server_config = config.clone();
|
let server_config = config.clone();
|
||||||
|
let server_policy_store = policy_store.clone();
|
||||||
|
|
||||||
let server_handle = tokio::spawn(async move {
|
let server_handle = tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
@@ -61,13 +68,15 @@ impl S3Server {
|
|||||||
let io = TokioIo::new(stream);
|
let io = TokioIo::new(stream);
|
||||||
let store = server_store.clone();
|
let store = server_store.clone();
|
||||||
let cfg = server_config.clone();
|
let cfg = server_config.clone();
|
||||||
|
let ps = server_policy_store.clone();
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let svc = service_fn(move |req: Request<Incoming>| {
|
let svc = service_fn(move |req: Request<Incoming>| {
|
||||||
let store = store.clone();
|
let store = store.clone();
|
||||||
let cfg = cfg.clone();
|
let cfg = cfg.clone();
|
||||||
|
let ps = ps.clone();
|
||||||
async move {
|
async move {
|
||||||
handle_request(req, store, cfg).await
|
handle_request(req, store, cfg, ps).await
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -95,12 +104,11 @@ impl S3Server {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if !config.server.silent {
|
if !config.server.silent {
|
||||||
tracing::info!("S3 server listening on {}", addr);
|
tracing::info!("Storage server listening on {}", addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
store,
|
store,
|
||||||
config,
|
|
||||||
shutdown_tx,
|
shutdown_tx,
|
||||||
server_handle,
|
server_handle,
|
||||||
})
|
})
|
||||||
@@ -116,7 +124,7 @@ impl S3Server {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl S3Config {
|
impl SmartStorageConfig {
|
||||||
fn address(&self) -> &str {
|
fn address(&self) -> &str {
|
||||||
&self.server.address
|
&self.server.address
|
||||||
}
|
}
|
||||||
@@ -184,7 +192,7 @@ fn empty_response(status: StatusCode, request_id: &str) -> Response<BoxBody> {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn s3_error_response(err: &S3Error, request_id: &str) -> Response<BoxBody> {
|
fn storage_error_response(err: &StorageError, request_id: &str) -> Response<BoxBody> {
|
||||||
let xml = err.to_xml();
|
let xml = err.to_xml();
|
||||||
Response::builder()
|
Response::builder()
|
||||||
.status(err.status)
|
.status(err.status)
|
||||||
@@ -197,7 +205,8 @@ fn s3_error_response(err: &S3Error, request_id: &str) -> Response<BoxBody> {
|
|||||||
async fn handle_request(
|
async fn handle_request(
|
||||||
req: Request<Incoming>,
|
req: Request<Incoming>,
|
||||||
store: Arc<FileStore>,
|
store: Arc<FileStore>,
|
||||||
config: S3Config,
|
config: SmartStorageConfig,
|
||||||
|
policy_store: Arc<PolicyStore>,
|
||||||
) -> Result<Response<BoxBody>, std::convert::Infallible> {
|
) -> Result<Response<BoxBody>, std::convert::Infallible> {
|
||||||
let request_id = Uuid::new_v4().to_string();
|
let request_id = Uuid::new_v4().to_string();
|
||||||
let method = req.method().clone();
|
let method = req.method().clone();
|
||||||
@@ -210,24 +219,49 @@ async fn handle_request(
|
|||||||
return Ok(resp);
|
return Ok(resp);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auth check
|
// Step 1: Resolve storage action from request
|
||||||
|
let request_ctx = action::resolve_action(&req);
|
||||||
|
|
||||||
|
// Step 2: Auth + policy pipeline
|
||||||
if config.auth.enabled {
|
if config.auth.enabled {
|
||||||
if let Err(e) = check_auth(&req, &config) {
|
// Attempt authentication
|
||||||
tracing::warn!("Auth failed: {}", e.message);
|
let identity = {
|
||||||
return Ok(s3_error_response(&e, &request_id));
|
let has_auth_header = req
|
||||||
|
.headers()
|
||||||
|
.get("authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|s| !s.is_empty())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if has_auth_header {
|
||||||
|
match auth::verify_request(&req, &config) {
|
||||||
|
Ok(id) => Some(id),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("Auth failed: {}", e.message);
|
||||||
|
return Ok(storage_error_response(&e, &request_id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None // Anonymous request
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Step 3: Authorization (policy evaluation)
|
||||||
|
if let Err(e) = authorize_request(&request_ctx, identity.as_ref(), &policy_store).await {
|
||||||
|
return Ok(storage_error_response(&e, &request_id));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Route and handle
|
// Route and handle
|
||||||
let mut response = match route_request(req, store, &config, &request_id).await {
|
let mut response = match route_request(req, store, &config, &request_id, &policy_store).await {
|
||||||
Ok(resp) => resp,
|
Ok(resp) => resp,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if let Some(s3err) = err.downcast_ref::<S3Error>() {
|
if let Some(s3err) = err.downcast_ref::<StorageError>() {
|
||||||
s3_error_response(s3err, &request_id)
|
storage_error_response(s3err, &request_id)
|
||||||
} else {
|
} else {
|
||||||
tracing::error!("Internal error: {}", err);
|
tracing::error!("Internal error: {}", err);
|
||||||
let s3err = S3Error::internal_error(&err.to_string());
|
let s3err = StorageError::internal_error(&err.to_string());
|
||||||
s3_error_response(&s3err, &request_id)
|
storage_error_response(&s3err, &request_id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -249,6 +283,42 @@ async fn handle_request(
|
|||||||
Ok(response)
|
Ok(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Authorize a request based on bucket policies and authentication state.
|
||||||
|
async fn authorize_request(
|
||||||
|
ctx: &RequestContext,
|
||||||
|
identity: Option<&AuthenticatedIdentity>,
|
||||||
|
policy_store: &PolicyStore,
|
||||||
|
) -> Result<(), StorageError> {
|
||||||
|
// ListAllMyBuckets requires authentication (no bucket to apply policy to)
|
||||||
|
if ctx.action == StorageAction::ListAllMyBuckets {
|
||||||
|
if identity.is_none() {
|
||||||
|
return Err(StorageError::access_denied());
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there's a bucket, check its policy
|
||||||
|
if let Some(ref bucket) = ctx.bucket {
|
||||||
|
if let Some(bucket_policy) = policy_store.get_policy(bucket).await {
|
||||||
|
let decision = policy::evaluate_policy(&bucket_policy, ctx, identity);
|
||||||
|
match decision {
|
||||||
|
PolicyDecision::Deny => return Err(StorageError::access_denied()),
|
||||||
|
PolicyDecision::Allow => return Ok(()),
|
||||||
|
PolicyDecision::NoOpinion => {
|
||||||
|
// Fall through to default behavior
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: authenticated users get full access, anonymous denied
|
||||||
|
if identity.is_none() {
|
||||||
|
return Err(StorageError::access_denied());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ============================
|
// ============================
|
||||||
// Routing
|
// Routing
|
||||||
// ============================
|
// ============================
|
||||||
@@ -256,8 +326,9 @@ async fn handle_request(
|
|||||||
async fn route_request(
|
async fn route_request(
|
||||||
req: Request<Incoming>,
|
req: Request<Incoming>,
|
||||||
store: Arc<FileStore>,
|
store: Arc<FileStore>,
|
||||||
_config: &S3Config,
|
_config: &SmartStorageConfig,
|
||||||
request_id: &str,
|
request_id: &str,
|
||||||
|
policy_store: &Arc<PolicyStore>,
|
||||||
) -> Result<Response<BoxBody>> {
|
) -> Result<Response<BoxBody>> {
|
||||||
let method = req.method().clone();
|
let method = req.method().clone();
|
||||||
let path = req.uri().path().to_string();
|
let path = req.uri().path().to_string();
|
||||||
@@ -282,6 +353,17 @@ async fn route_request(
|
|||||||
1 => {
|
1 => {
|
||||||
// Bucket level: /{bucket}
|
// Bucket level: /{bucket}
|
||||||
let bucket = percent_decode(segments[0]);
|
let bucket = percent_decode(segments[0]);
|
||||||
|
|
||||||
|
// Check for ?policy query parameter
|
||||||
|
if query.contains_key("policy") {
|
||||||
|
return match method {
|
||||||
|
Method::GET => handle_get_bucket_policy(policy_store, &bucket, request_id).await,
|
||||||
|
Method::PUT => handle_put_bucket_policy(req, &store, policy_store, &bucket, request_id).await,
|
||||||
|
Method::DELETE => handle_delete_bucket_policy(policy_store, &bucket, request_id).await,
|
||||||
|
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
match method {
|
match method {
|
||||||
Method::GET => {
|
Method::GET => {
|
||||||
if query.contains_key("uploads") {
|
if query.contains_key("uploads") {
|
||||||
@@ -291,7 +373,7 @@ async fn route_request(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::PUT => handle_create_bucket(store, &bucket, request_id).await,
|
Method::PUT => handle_create_bucket(store, &bucket, request_id).await,
|
||||||
Method::DELETE => handle_delete_bucket(store, &bucket, request_id).await,
|
Method::DELETE => handle_delete_bucket(store, &bucket, request_id, policy_store).await,
|
||||||
Method::HEAD => handle_head_bucket(store, &bucket, request_id).await,
|
Method::HEAD => handle_head_bucket(store, &bucket, request_id).await,
|
||||||
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||||
}
|
}
|
||||||
@@ -332,8 +414,8 @@ async fn route_request(
|
|||||||
let upload_id = query.get("uploadId").unwrap().clone();
|
let upload_id = query.get("uploadId").unwrap().clone();
|
||||||
handle_complete_multipart(req, store, &bucket, &key, &upload_id, request_id).await
|
handle_complete_multipart(req, store, &bucket, &key, &upload_id, request_id).await
|
||||||
} else {
|
} else {
|
||||||
let err = S3Error::invalid_request("Invalid POST request");
|
let err = StorageError::invalid_request("Invalid POST request");
|
||||||
Ok(s3_error_response(&err, request_id))
|
Ok(storage_error_response(&err, request_id))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||||
@@ -369,8 +451,11 @@ async fn handle_delete_bucket(
|
|||||||
store: Arc<FileStore>,
|
store: Arc<FileStore>,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
request_id: &str,
|
request_id: &str,
|
||||||
|
policy_store: &Arc<PolicyStore>,
|
||||||
) -> Result<Response<BoxBody>> {
|
) -> Result<Response<BoxBody>> {
|
||||||
store.delete_bucket(bucket).await?;
|
store.delete_bucket(bucket).await?;
|
||||||
|
// Clean up bucket policy on deletion
|
||||||
|
let _ = policy_store.delete_policy(bucket).await;
|
||||||
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,7 +467,7 @@ async fn handle_head_bucket(
|
|||||||
if store.bucket_exists(bucket).await {
|
if store.bucket_exists(bucket).await {
|
||||||
Ok(empty_response(StatusCode::OK, request_id))
|
Ok(empty_response(StatusCode::OK, request_id))
|
||||||
} else {
|
} else {
|
||||||
Err(S3Error::no_such_bucket().into())
|
Err(StorageError::no_such_bucket().into())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -577,6 +662,70 @@ async fn handle_copy_object(
|
|||||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Policy handlers
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
async fn handle_get_bucket_policy(
|
||||||
|
policy_store: &Arc<PolicyStore>,
|
||||||
|
bucket: &str,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<Response<BoxBody>> {
|
||||||
|
match policy_store.get_policy(bucket).await {
|
||||||
|
Some(p) => {
|
||||||
|
let json = serde_json::to_string_pretty(&p)?;
|
||||||
|
let resp = Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header("content-type", "application/json")
|
||||||
|
.header("x-amz-request-id", request_id)
|
||||||
|
.body(full_body(json))
|
||||||
|
.unwrap();
|
||||||
|
Ok(resp)
|
||||||
|
}
|
||||||
|
None => Err(StorageError::no_such_bucket_policy().into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_put_bucket_policy(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
store: &Arc<FileStore>,
|
||||||
|
policy_store: &Arc<PolicyStore>,
|
||||||
|
bucket: &str,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<Response<BoxBody>> {
|
||||||
|
// Verify bucket exists
|
||||||
|
if !store.bucket_exists(bucket).await {
|
||||||
|
return Err(StorageError::no_such_bucket().into());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read body
|
||||||
|
let body_bytes = req.collect().await.map_err(|e| anyhow::anyhow!("Body error: {}", e))?.to_bytes();
|
||||||
|
let body_str = String::from_utf8_lossy(&body_bytes);
|
||||||
|
|
||||||
|
// Validate and parse
|
||||||
|
let validated_policy = policy::validate_policy(&body_str)?;
|
||||||
|
|
||||||
|
// Store
|
||||||
|
policy_store
|
||||||
|
.put_policy(bucket, validated_policy)
|
||||||
|
.await
|
||||||
|
.map_err(|e| StorageError::internal_error(&e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_delete_bucket_policy(
|
||||||
|
policy_store: &Arc<PolicyStore>,
|
||||||
|
bucket: &str,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<Response<BoxBody>> {
|
||||||
|
policy_store
|
||||||
|
.delete_policy(bucket)
|
||||||
|
.await
|
||||||
|
.map_err(|e| StorageError::internal_error(&e.to_string()))?;
|
||||||
|
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||||
|
}
|
||||||
|
|
||||||
// ============================
|
// ============================
|
||||||
// Multipart handlers
|
// Multipart handlers
|
||||||
// ============================
|
// ============================
|
||||||
@@ -607,7 +756,7 @@ async fn handle_upload_part(
|
|||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
if part_number < 1 || part_number > 10000 {
|
if part_number < 1 || part_number > 10000 {
|
||||||
return Err(S3Error::invalid_part_number().into());
|
return Err(StorageError::invalid_part_number().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let body = req.into_body();
|
let body = req.into_body();
|
||||||
@@ -776,7 +925,7 @@ fn extract_xml_value<'a>(xml: &'a str, tag: &str) -> Option<String> {
|
|||||||
// CORS
|
// CORS
|
||||||
// ============================
|
// ============================
|
||||||
|
|
||||||
fn build_cors_preflight(config: &S3Config, request_id: &str) -> Response<BoxBody> {
|
fn build_cors_preflight(config: &SmartStorageConfig, request_id: &str) -> Response<BoxBody> {
|
||||||
let mut builder = Response::builder()
|
let mut builder = Response::builder()
|
||||||
.status(StatusCode::NO_CONTENT)
|
.status(StatusCode::NO_CONTENT)
|
||||||
.header("x-amz-request-id", request_id);
|
.header("x-amz-request-id", request_id);
|
||||||
@@ -800,7 +949,7 @@ fn build_cors_preflight(config: &S3Config, request_id: &str) -> Response<BoxBody
|
|||||||
builder.body(empty_body()).unwrap()
|
builder.body(empty_body()).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &S3Config) {
|
fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &SmartStorageConfig) {
|
||||||
if let Some(ref origins) = config.cors.allowed_origins {
|
if let Some(ref origins) = config.cors.allowed_origins {
|
||||||
headers.insert(
|
headers.insert(
|
||||||
"access-control-allow-origin",
|
"access-control-allow-origin",
|
||||||
@@ -820,46 +969,3 @@ fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &S3Config) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================
|
|
||||||
// Auth
|
|
||||||
// ============================
|
|
||||||
|
|
||||||
fn check_auth(req: &Request<Incoming>, config: &S3Config) -> Result<(), S3Error> {
|
|
||||||
let auth_header = req
|
|
||||||
.headers()
|
|
||||||
.get("authorization")
|
|
||||||
.and_then(|v| v.to_str().ok())
|
|
||||||
.unwrap_or("");
|
|
||||||
|
|
||||||
if auth_header.is_empty() {
|
|
||||||
return Err(S3Error::access_denied());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract access key from AWS v2 or v4 signature
|
|
||||||
let access_key = if auth_header.starts_with("AWS4-HMAC-SHA256") {
|
|
||||||
// v4: AWS4-HMAC-SHA256 Credential=KEY/date/region/s3/aws4_request, ...
|
|
||||||
auth_header
|
|
||||||
.split("Credential=")
|
|
||||||
.nth(1)
|
|
||||||
.and_then(|s| s.split('/').next())
|
|
||||||
} else if auth_header.starts_with("AWS ") {
|
|
||||||
// v2: AWS KEY:signature
|
|
||||||
auth_header
|
|
||||||
.strip_prefix("AWS ")
|
|
||||||
.and_then(|s| s.split(':').next())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
let access_key = access_key.unwrap_or("");
|
|
||||||
|
|
||||||
// Check against configured credentials
|
|
||||||
for cred in &config.auth.credentials {
|
|
||||||
if cred.access_key_id == access_key {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Err(S3Error::access_denied())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -10,19 +10,17 @@ use tokio::fs;
|
|||||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufWriter};
|
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufWriter};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::s3_error::S3Error;
|
use crate::error::StorageError;
|
||||||
|
|
||||||
// ============================
|
// ============================
|
||||||
// Result types
|
// Result types
|
||||||
// ============================
|
// ============================
|
||||||
|
|
||||||
pub struct PutResult {
|
pub struct PutResult {
|
||||||
pub size: u64,
|
|
||||||
pub md5: String,
|
pub md5: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct GetResult {
|
pub struct GetResult {
|
||||||
pub key: String,
|
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
pub last_modified: DateTime<Utc>,
|
pub last_modified: DateTime<Utc>,
|
||||||
pub md5: String,
|
pub md5: String,
|
||||||
@@ -32,7 +30,6 @@ pub struct GetResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct HeadResult {
|
pub struct HeadResult {
|
||||||
pub key: String,
|
|
||||||
pub size: u64,
|
pub size: u64,
|
||||||
pub last_modified: DateTime<Utc>,
|
pub last_modified: DateTime<Utc>,
|
||||||
pub md5: String,
|
pub md5: String,
|
||||||
@@ -40,7 +37,6 @@ pub struct HeadResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub struct CopyResult {
|
pub struct CopyResult {
|
||||||
pub size: u64,
|
|
||||||
pub md5: String,
|
pub md5: String,
|
||||||
pub last_modified: DateTime<Utc>,
|
pub last_modified: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
@@ -69,14 +65,12 @@ pub struct BucketInfo {
|
|||||||
|
|
||||||
pub struct MultipartUploadInfo {
|
pub struct MultipartUploadInfo {
|
||||||
pub upload_id: String,
|
pub upload_id: String,
|
||||||
pub bucket: String,
|
|
||||||
pub key: String,
|
pub key: String,
|
||||||
pub initiated: DateTime<Utc>,
|
pub initiated: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct CompleteMultipartResult {
|
pub struct CompleteMultipartResult {
|
||||||
pub etag: String,
|
pub etag: String,
|
||||||
pub size: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================
|
// ============================
|
||||||
@@ -118,14 +112,20 @@ impl FileStore {
|
|||||||
|
|
||||||
pub async fn initialize(&self) -> Result<()> {
|
pub async fn initialize(&self) -> Result<()> {
|
||||||
fs::create_dir_all(&self.root_dir).await?;
|
fs::create_dir_all(&self.root_dir).await?;
|
||||||
|
fs::create_dir_all(self.policies_dir()).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn policies_dir(&self) -> PathBuf {
|
||||||
|
self.root_dir.join(".policies")
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn reset(&self) -> Result<()> {
|
pub async fn reset(&self) -> Result<()> {
|
||||||
if self.root_dir.exists() {
|
if self.root_dir.exists() {
|
||||||
fs::remove_dir_all(&self.root_dir).await?;
|
fs::remove_dir_all(&self.root_dir).await?;
|
||||||
}
|
}
|
||||||
fs::create_dir_all(&self.root_dir).await?;
|
fs::create_dir_all(&self.root_dir).await?;
|
||||||
|
fs::create_dir_all(self.policies_dir()).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,13 +174,13 @@ impl FileStore {
|
|||||||
let bucket_path = self.root_dir.join(bucket);
|
let bucket_path = self.root_dir.join(bucket);
|
||||||
|
|
||||||
if !bucket_path.is_dir() {
|
if !bucket_path.is_dir() {
|
||||||
return Err(S3Error::no_such_bucket().into());
|
return Err(StorageError::no_such_bucket().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if bucket is empty (ignore hidden files)
|
// Check if bucket is empty (ignore hidden files)
|
||||||
let mut entries = fs::read_dir(&bucket_path).await?;
|
let mut entries = fs::read_dir(&bucket_path).await?;
|
||||||
while let Some(_entry) = entries.next_entry().await? {
|
while let Some(_entry) = entries.next_entry().await? {
|
||||||
return Err(S3Error::bucket_not_empty().into());
|
return Err(StorageError::bucket_not_empty().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
fs::remove_dir_all(&bucket_path).await?;
|
fs::remove_dir_all(&bucket_path).await?;
|
||||||
@@ -199,7 +199,7 @@ impl FileStore {
|
|||||||
metadata: HashMap<String, String>,
|
metadata: HashMap<String, String>,
|
||||||
) -> Result<PutResult> {
|
) -> Result<PutResult> {
|
||||||
if !self.bucket_exists(bucket).await {
|
if !self.bucket_exists(bucket).await {
|
||||||
return Err(S3Error::no_such_bucket().into());
|
return Err(StorageError::no_such_bucket().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let object_path = self.object_path(bucket, key);
|
let object_path = self.object_path(bucket, key);
|
||||||
@@ -210,7 +210,6 @@ impl FileStore {
|
|||||||
let file = fs::File::create(&object_path).await?;
|
let file = fs::File::create(&object_path).await?;
|
||||||
let mut writer = BufWriter::new(file);
|
let mut writer = BufWriter::new(file);
|
||||||
let mut hasher = Md5::new();
|
let mut hasher = Md5::new();
|
||||||
let mut total_size: u64 = 0;
|
|
||||||
|
|
||||||
// Stream body frames directly to file
|
// Stream body frames directly to file
|
||||||
let mut body = body;
|
let mut body = body;
|
||||||
@@ -219,7 +218,6 @@ impl FileStore {
|
|||||||
Some(Ok(frame)) => {
|
Some(Ok(frame)) => {
|
||||||
if let Ok(data) = frame.into_data() {
|
if let Ok(data) = frame.into_data() {
|
||||||
hasher.update(&data);
|
hasher.update(&data);
|
||||||
total_size += data.len() as u64;
|
|
||||||
writer.write_all(&data).await?;
|
writer.write_all(&data).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -245,44 +243,6 @@ impl FileStore {
|
|||||||
fs::write(&metadata_path, metadata_json).await?;
|
fs::write(&metadata_path, metadata_json).await?;
|
||||||
|
|
||||||
Ok(PutResult {
|
Ok(PutResult {
|
||||||
size: total_size,
|
|
||||||
md5: md5_hex,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn put_object_bytes(
|
|
||||||
&self,
|
|
||||||
bucket: &str,
|
|
||||||
key: &str,
|
|
||||||
data: &[u8],
|
|
||||||
metadata: HashMap<String, String>,
|
|
||||||
) -> Result<PutResult> {
|
|
||||||
if !self.bucket_exists(bucket).await {
|
|
||||||
return Err(S3Error::no_such_bucket().into());
|
|
||||||
}
|
|
||||||
|
|
||||||
let object_path = self.object_path(bucket, key);
|
|
||||||
if let Some(parent) = object_path.parent() {
|
|
||||||
fs::create_dir_all(parent).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut hasher = Md5::new();
|
|
||||||
hasher.update(data);
|
|
||||||
let md5_hex = format!("{:x}", hasher.finalize());
|
|
||||||
|
|
||||||
fs::write(&object_path, data).await?;
|
|
||||||
|
|
||||||
// Write MD5 sidecar
|
|
||||||
let md5_path = format!("{}.md5", object_path.display());
|
|
||||||
fs::write(&md5_path, &md5_hex).await?;
|
|
||||||
|
|
||||||
// Write metadata sidecar
|
|
||||||
let metadata_path = format!("{}.metadata.json", object_path.display());
|
|
||||||
let metadata_json = serde_json::to_string_pretty(&metadata)?;
|
|
||||||
fs::write(&metadata_path, metadata_json).await?;
|
|
||||||
|
|
||||||
Ok(PutResult {
|
|
||||||
size: data.len() as u64,
|
|
||||||
md5: md5_hex,
|
md5: md5_hex,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -296,7 +256,7 @@ impl FileStore {
|
|||||||
let object_path = self.object_path(bucket, key);
|
let object_path = self.object_path(bucket, key);
|
||||||
|
|
||||||
if !object_path.exists() {
|
if !object_path.exists() {
|
||||||
return Err(S3Error::no_such_key().into());
|
return Err(StorageError::no_such_key().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let file_meta = fs::metadata(&object_path).await?;
|
let file_meta = fs::metadata(&object_path).await?;
|
||||||
@@ -316,7 +276,6 @@ impl FileStore {
|
|||||||
};
|
};
|
||||||
|
|
||||||
Ok(GetResult {
|
Ok(GetResult {
|
||||||
key: key.to_string(),
|
|
||||||
size,
|
size,
|
||||||
last_modified,
|
last_modified,
|
||||||
md5,
|
md5,
|
||||||
@@ -330,7 +289,7 @@ impl FileStore {
|
|||||||
let object_path = self.object_path(bucket, key);
|
let object_path = self.object_path(bucket, key);
|
||||||
|
|
||||||
if !object_path.exists() {
|
if !object_path.exists() {
|
||||||
return Err(S3Error::no_such_key().into());
|
return Err(StorageError::no_such_key().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only stat the file, don't open it
|
// Only stat the file, don't open it
|
||||||
@@ -342,7 +301,6 @@ impl FileStore {
|
|||||||
let metadata = self.read_metadata(&object_path).await;
|
let metadata = self.read_metadata(&object_path).await;
|
||||||
|
|
||||||
Ok(HeadResult {
|
Ok(HeadResult {
|
||||||
key: key.to_string(),
|
|
||||||
size,
|
size,
|
||||||
last_modified,
|
last_modified,
|
||||||
md5,
|
md5,
|
||||||
@@ -394,11 +352,11 @@ impl FileStore {
|
|||||||
let dest_path = self.object_path(dest_bucket, dest_key);
|
let dest_path = self.object_path(dest_bucket, dest_key);
|
||||||
|
|
||||||
if !src_path.exists() {
|
if !src_path.exists() {
|
||||||
return Err(S3Error::no_such_key().into());
|
return Err(StorageError::no_such_key().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
if !self.bucket_exists(dest_bucket).await {
|
if !self.bucket_exists(dest_bucket).await {
|
||||||
return Err(S3Error::no_such_bucket().into());
|
return Err(StorageError::no_such_bucket().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(parent) = dest_path.parent() {
|
if let Some(parent) = dest_path.parent() {
|
||||||
@@ -429,7 +387,6 @@ impl FileStore {
|
|||||||
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
|
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
|
||||||
|
|
||||||
Ok(CopyResult {
|
Ok(CopyResult {
|
||||||
size: file_meta.len(),
|
|
||||||
md5,
|
md5,
|
||||||
last_modified,
|
last_modified,
|
||||||
})
|
})
|
||||||
@@ -446,7 +403,7 @@ impl FileStore {
|
|||||||
let bucket_path = self.root_dir.join(bucket);
|
let bucket_path = self.root_dir.join(bucket);
|
||||||
|
|
||||||
if !bucket_path.is_dir() {
|
if !bucket_path.is_dir() {
|
||||||
return Err(S3Error::no_such_bucket().into());
|
return Err(StorageError::no_such_bucket().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collect all object keys recursively
|
// Collect all object keys recursively
|
||||||
@@ -571,7 +528,7 @@ impl FileStore {
|
|||||||
) -> Result<(String, u64)> {
|
) -> Result<(String, u64)> {
|
||||||
let upload_dir = self.multipart_dir().join(upload_id);
|
let upload_dir = self.multipart_dir().join(upload_id);
|
||||||
if !upload_dir.is_dir() {
|
if !upload_dir.is_dir() {
|
||||||
return Err(S3Error::no_such_upload().into());
|
return Err(StorageError::no_such_upload().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let part_path = upload_dir.join(format!("part-{}", part_number));
|
let part_path = upload_dir.join(format!("part-{}", part_number));
|
||||||
@@ -645,7 +602,7 @@ impl FileStore {
|
|||||||
) -> Result<CompleteMultipartResult> {
|
) -> Result<CompleteMultipartResult> {
|
||||||
let upload_dir = self.multipart_dir().join(upload_id);
|
let upload_dir = self.multipart_dir().join(upload_id);
|
||||||
if !upload_dir.is_dir() {
|
if !upload_dir.is_dir() {
|
||||||
return Err(S3Error::no_such_upload().into());
|
return Err(StorageError::no_such_upload().into());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read metadata to get bucket/key
|
// Read metadata to get bucket/key
|
||||||
@@ -662,7 +619,6 @@ impl FileStore {
|
|||||||
let dest_file = fs::File::create(&object_path).await?;
|
let dest_file = fs::File::create(&object_path).await?;
|
||||||
let mut writer = BufWriter::new(dest_file);
|
let mut writer = BufWriter::new(dest_file);
|
||||||
let mut hasher = Md5::new();
|
let mut hasher = Md5::new();
|
||||||
let mut total_size: u64 = 0;
|
|
||||||
|
|
||||||
for (part_number, _etag) in parts {
|
for (part_number, _etag) in parts {
|
||||||
let part_path = upload_dir.join(format!("part-{}", part_number));
|
let part_path = upload_dir.join(format!("part-{}", part_number));
|
||||||
@@ -679,7 +635,6 @@ impl FileStore {
|
|||||||
}
|
}
|
||||||
hasher.update(&buf[..n]);
|
hasher.update(&buf[..n]);
|
||||||
writer.write_all(&buf[..n]).await?;
|
writer.write_all(&buf[..n]).await?;
|
||||||
total_size += n as u64;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -702,14 +657,13 @@ impl FileStore {
|
|||||||
|
|
||||||
Ok(CompleteMultipartResult {
|
Ok(CompleteMultipartResult {
|
||||||
etag,
|
etag,
|
||||||
size: total_size,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn abort_multipart(&self, upload_id: &str) -> Result<()> {
|
pub async fn abort_multipart(&self, upload_id: &str) -> Result<()> {
|
||||||
let upload_dir = self.multipart_dir().join(upload_id);
|
let upload_dir = self.multipart_dir().join(upload_id);
|
||||||
if !upload_dir.is_dir() {
|
if !upload_dir.is_dir() {
|
||||||
return Err(S3Error::no_such_upload().into());
|
return Err(StorageError::no_such_upload().into());
|
||||||
}
|
}
|
||||||
fs::remove_dir_all(&upload_dir).await?;
|
fs::remove_dir_all(&upload_dir).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -742,7 +696,6 @@ impl FileStore {
|
|||||||
|
|
||||||
uploads.push(MultipartUploadInfo {
|
uploads.push(MultipartUploadInfo {
|
||||||
upload_id: meta.upload_id,
|
upload_id: meta.upload_id,
|
||||||
bucket: meta.bucket,
|
|
||||||
key: meta.key,
|
key: meta.key,
|
||||||
initiated,
|
initiated,
|
||||||
});
|
});
|
||||||
@@ -762,7 +715,7 @@ impl FileStore {
|
|||||||
let encoded = encode_key(key);
|
let encoded = encode_key(key);
|
||||||
self.root_dir
|
self.root_dir
|
||||||
.join(bucket)
|
.join(bucket)
|
||||||
.join(format!("{}._S3_object", encoded))
|
.join(format!("{}._storage_object", encoded))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_md5(&self, object_path: &Path) -> String {
|
async fn read_md5(&self, object_path: &Path) -> String {
|
||||||
@@ -822,7 +775,7 @@ impl FileStore {
|
|||||||
|
|
||||||
if meta.is_dir() {
|
if meta.is_dir() {
|
||||||
self.collect_keys(bucket_path, &entry.path(), keys).await?;
|
self.collect_keys(bucket_path, &entry.path(), keys).await?;
|
||||||
} else if name.ends_with("._S3_object")
|
} else if name.ends_with("._storage_object")
|
||||||
&& !name.ends_with(".metadata.json")
|
&& !name.ends_with(".metadata.json")
|
||||||
&& !name.ends_with(".md5")
|
&& !name.ends_with(".md5")
|
||||||
{
|
{
|
||||||
@@ -832,7 +785,7 @@ impl FileStore {
|
|||||||
.unwrap_or(Path::new(""))
|
.unwrap_or(Path::new(""))
|
||||||
.to_string_lossy()
|
.to_string_lossy()
|
||||||
.to_string();
|
.to_string();
|
||||||
let key = decode_key(relative.trim_end_matches("._S3_object"));
|
let key = decode_key(relative.trim_end_matches("._storage_object"));
|
||||||
keys.push(key);
|
keys.push(key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use crate::storage::{BucketInfo, ListObjectsResult, MultipartUploadInfo};
|
use crate::storage::{BucketInfo, ListObjectsResult, MultipartUploadInfo};
|
||||||
|
|
||||||
const XML_DECL: &str = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
|
const XML_DECL: &str = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
|
||||||
const S3_NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
|
const STORAGE_NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
|
||||||
|
|
||||||
fn xml_escape(s: &str) -> String {
|
fn xml_escape(s: &str) -> String {
|
||||||
s.replace('&', "&")
|
s.replace('&', "&")
|
||||||
@@ -14,9 +14,9 @@ fn xml_escape(s: &str) -> String {
|
|||||||
pub fn list_buckets_xml(buckets: &[BucketInfo]) -> String {
|
pub fn list_buckets_xml(buckets: &[BucketInfo]) -> String {
|
||||||
let mut xml = format!(
|
let mut xml = format!(
|
||||||
"{}\n<ListAllMyBucketsResult xmlns=\"{}\">\
|
"{}\n<ListAllMyBucketsResult xmlns=\"{}\">\
|
||||||
<Owner><ID>123456789000</ID><DisplayName>S3rver</DisplayName></Owner>\
|
<Owner><ID>123456789000</ID><DisplayName>Storage</DisplayName></Owner>\
|
||||||
<Buckets>",
|
<Buckets>",
|
||||||
XML_DECL, S3_NS
|
XML_DECL, STORAGE_NS
|
||||||
);
|
);
|
||||||
|
|
||||||
for b in buckets {
|
for b in buckets {
|
||||||
@@ -39,7 +39,7 @@ pub fn list_objects_v1_xml(bucket: &str, result: &ListObjectsResult) -> String {
|
|||||||
<MaxKeys>{}</MaxKeys>\
|
<MaxKeys>{}</MaxKeys>\
|
||||||
<IsTruncated>{}</IsTruncated>",
|
<IsTruncated>{}</IsTruncated>",
|
||||||
XML_DECL,
|
XML_DECL,
|
||||||
S3_NS,
|
STORAGE_NS,
|
||||||
xml_escape(bucket),
|
xml_escape(bucket),
|
||||||
xml_escape(&result.prefix),
|
xml_escape(&result.prefix),
|
||||||
result.max_keys,
|
result.max_keys,
|
||||||
@@ -86,7 +86,7 @@ pub fn list_objects_v2_xml(bucket: &str, result: &ListObjectsResult) -> String {
|
|||||||
<KeyCount>{}</KeyCount>\
|
<KeyCount>{}</KeyCount>\
|
||||||
<IsTruncated>{}</IsTruncated>",
|
<IsTruncated>{}</IsTruncated>",
|
||||||
XML_DECL,
|
XML_DECL,
|
||||||
S3_NS,
|
STORAGE_NS,
|
||||||
xml_escape(bucket),
|
xml_escape(bucket),
|
||||||
xml_escape(&result.prefix),
|
xml_escape(&result.prefix),
|
||||||
result.max_keys,
|
result.max_keys,
|
||||||
@@ -132,15 +132,6 @@ pub fn list_objects_v2_xml(bucket: &str, result: &ListObjectsResult) -> String {
|
|||||||
xml
|
xml
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn error_xml(code: &str, message: &str) -> String {
|
|
||||||
format!(
|
|
||||||
"{}\n<Error><Code>{}</Code><Message>{}</Message></Error>",
|
|
||||||
XML_DECL,
|
|
||||||
xml_escape(code),
|
|
||||||
xml_escape(message)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
|
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{}\n<CopyObjectResult>\
|
"{}\n<CopyObjectResult>\
|
||||||
@@ -161,7 +152,7 @@ pub fn initiate_multipart_xml(bucket: &str, key: &str, upload_id: &str) -> Strin
|
|||||||
<UploadId>{}</UploadId>\
|
<UploadId>{}</UploadId>\
|
||||||
</InitiateMultipartUploadResult>",
|
</InitiateMultipartUploadResult>",
|
||||||
XML_DECL,
|
XML_DECL,
|
||||||
S3_NS,
|
STORAGE_NS,
|
||||||
xml_escape(bucket),
|
xml_escape(bucket),
|
||||||
xml_escape(key),
|
xml_escape(key),
|
||||||
xml_escape(upload_id)
|
xml_escape(upload_id)
|
||||||
@@ -177,7 +168,7 @@ pub fn complete_multipart_xml(bucket: &str, key: &str, etag: &str) -> String {
|
|||||||
<ETag>\"{}\"</ETag>\
|
<ETag>\"{}\"</ETag>\
|
||||||
</CompleteMultipartUploadResult>",
|
</CompleteMultipartUploadResult>",
|
||||||
XML_DECL,
|
XML_DECL,
|
||||||
S3_NS,
|
STORAGE_NS,
|
||||||
xml_escape(bucket),
|
xml_escape(bucket),
|
||||||
xml_escape(key),
|
xml_escape(key),
|
||||||
xml_escape(bucket),
|
xml_escape(bucket),
|
||||||
@@ -195,7 +186,7 @@ pub fn list_multipart_uploads_xml(bucket: &str, uploads: &[MultipartUploadInfo])
|
|||||||
<MaxUploads>1000</MaxUploads>\
|
<MaxUploads>1000</MaxUploads>\
|
||||||
<IsTruncated>false</IsTruncated>",
|
<IsTruncated>false</IsTruncated>",
|
||||||
XML_DECL,
|
XML_DECL,
|
||||||
S3_NS,
|
STORAGE_NS,
|
||||||
xml_escape(bucket)
|
xml_escape(bucket)
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -204,8 +195,8 @@ pub fn list_multipart_uploads_xml(bucket: &str, uploads: &[MultipartUploadInfo])
|
|||||||
"<Upload>\
|
"<Upload>\
|
||||||
<Key>{}</Key>\
|
<Key>{}</Key>\
|
||||||
<UploadId>{}</UploadId>\
|
<UploadId>{}</UploadId>\
|
||||||
<Initiator><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Initiator>\
|
<Initiator><ID>STORAGE</ID><DisplayName>STORAGE</DisplayName></Initiator>\
|
||||||
<Owner><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Owner>\
|
<Owner><ID>STORAGE</ID><DisplayName>STORAGE</DisplayName></Owner>\
|
||||||
<StorageClass>STANDARD</StorageClass>\
|
<StorageClass>STANDARD</StorageClass>\
|
||||||
<Initiated>{}</Initiated>\
|
<Initiated>{}</Initiated>\
|
||||||
</Upload>",
|
</Upload>",
|
||||||
|
|||||||
301
test/test.auth.node.ts
Normal file
301
test/test.auth.node.ts
Normal file
@@ -0,0 +1,301 @@
|
|||||||
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
CreateBucketCommand,
|
||||||
|
ListBucketsCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
DeleteBucketCommand,
|
||||||
|
PutBucketPolicyCommand,
|
||||||
|
GetBucketPolicyCommand,
|
||||||
|
DeleteBucketPolicyCommand,
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
|
let authClient: S3Client;
|
||||||
|
let wrongClient: S3Client;
|
||||||
|
|
||||||
|
const TEST_PORT = 3344;
|
||||||
|
const ACCESS_KEY = 'TESTAKID';
|
||||||
|
const SECRET_KEY = 'TESTSECRETKEY123';
|
||||||
|
|
||||||
|
async function streamToString(stream: Readable): Promise<string> {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
||||||
|
stream.on('error', reject);
|
||||||
|
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Server setup
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('should start storage server with auth enabled', async () => {
|
||||||
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
|
server: {
|
||||||
|
port: TEST_PORT,
|
||||||
|
silent: true,
|
||||||
|
region: 'us-east-1',
|
||||||
|
},
|
||||||
|
storage: {
|
||||||
|
cleanSlate: true,
|
||||||
|
},
|
||||||
|
auth: {
|
||||||
|
enabled: true,
|
||||||
|
credentials: [
|
||||||
|
{
|
||||||
|
accessKeyId: ACCESS_KEY,
|
||||||
|
secretAccessKey: SECRET_KEY,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Authenticated client with correct credentials
|
||||||
|
authClient = new S3Client({
|
||||||
|
endpoint: `http://localhost:${TEST_PORT}`,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: ACCESS_KEY,
|
||||||
|
secretAccessKey: SECRET_KEY,
|
||||||
|
},
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Client with wrong credentials
|
||||||
|
wrongClient = new S3Client({
|
||||||
|
endpoint: `http://localhost:${TEST_PORT}`,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: 'WRONGKEY',
|
||||||
|
secretAccessKey: 'WRONGSECRET',
|
||||||
|
},
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Authenticated CRUD
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('authenticated: should list buckets', async () => {
|
||||||
|
const response = await authClient.send(new ListBucketsCommand({}));
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
expect(Array.isArray(response.Buckets)).toEqual(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('authenticated: should create a bucket', async () => {
|
||||||
|
const response = await authClient.send(new CreateBucketCommand({ Bucket: 'auth-test-bucket' }));
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('authenticated: should upload an object', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Key: 'hello.txt',
|
||||||
|
Body: 'Hello authenticated world!',
|
||||||
|
ContentType: 'text/plain',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('authenticated: should download the object', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new GetObjectCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Key: 'hello.txt',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
const content = await streamToString(response.Body as Readable);
|
||||||
|
expect(content).toEqual('Hello authenticated world!');
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Wrong credentials → 403
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('wrong credentials: should fail to list buckets', async () => {
|
||||||
|
await expect(wrongClient.send(new ListBucketsCommand({}))).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('wrong credentials: should fail to get object', async () => {
|
||||||
|
await expect(
|
||||||
|
wrongClient.send(
|
||||||
|
new GetObjectCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Key: 'hello.txt',
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Anonymous → 403 (no policy yet)
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('anonymous: should fail to list buckets', async () => {
|
||||||
|
const resp = await fetch(`http://localhost:${TEST_PORT}/`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('anonymous: should fail to get object (no policy)', async () => {
|
||||||
|
const resp = await fetch(`http://localhost:${TEST_PORT}/auth-test-bucket/hello.txt`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Bucket policy: public read
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('should PUT a public-read bucket policy', async () => {
|
||||||
|
const policy = {
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: 'PublicRead',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject'],
|
||||||
|
Resource: [`arn:aws:s3:::auth-test-bucket/*`],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Policy: JSON.stringify(policy),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should GET the bucket policy', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new GetBucketPolicyCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
const policy = JSON.parse(response.Policy!);
|
||||||
|
expect(policy.Statement[0].Sid).toEqual('PublicRead');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('anonymous: should GET object after public-read policy', async () => {
|
||||||
|
const resp = await fetch(`http://localhost:${TEST_PORT}/auth-test-bucket/hello.txt`);
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
const content = await resp.text();
|
||||||
|
expect(content).toEqual('Hello authenticated world!');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('anonymous: should still fail to PUT object (policy only allows GET)', async () => {
|
||||||
|
const resp = await fetch(`http://localhost:${TEST_PORT}/auth-test-bucket/anon-file.txt`, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'should fail',
|
||||||
|
});
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Deny policy
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('should PUT a deny policy that blocks authenticated delete', async () => {
|
||||||
|
const policy = {
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: 'PublicRead',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject'],
|
||||||
|
Resource: [`arn:aws:s3:::auth-test-bucket/*`],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: 'DenyDelete',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:DeleteObject'],
|
||||||
|
Resource: [`arn:aws:s3:::auth-test-bucket/*`],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Policy: JSON.stringify(policy),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('authenticated: should be denied delete by policy', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new DeleteObjectCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Key: 'hello.txt',
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// DELETE bucket policy
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('should DELETE the bucket policy', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new DeleteBucketPolicyCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should GET policy → 404 after deletion', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new GetBucketPolicyCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Cleanup
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('authenticated: delete object after policy removed', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new DeleteObjectCommand({
|
||||||
|
Bucket: 'auth-test-bucket',
|
||||||
|
Key: 'hello.txt',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('authenticated: delete the bucket', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new DeleteBucketCommand({ Bucket: 'auth-test-bucket' }),
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('should stop the storage server', async () => {
|
||||||
|
await testSmartStorageInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
|
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
import * as smarts3 from '../ts/index.js';
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
let testSmarts3Instance: smarts3.Smarts3;
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
let s3Client: S3Client;
|
let s3Client: S3Client;
|
||||||
|
|
||||||
// Helper to convert stream to string
|
// Helper to convert stream to string
|
||||||
@@ -16,8 +16,8 @@ async function streamToString(stream: Readable): Promise<string> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
tap.test('should start the S3 server and configure client', async () => {
|
tap.test('should start the storage server and configure client', async () => {
|
||||||
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
server: {
|
server: {
|
||||||
port: 3337,
|
port: 3337,
|
||||||
silent: true,
|
silent: true,
|
||||||
@@ -27,7 +27,7 @@ tap.test('should start the S3 server and configure client', async () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const descriptor = await testSmarts3Instance.getS3Descriptor();
|
const descriptor = await testSmartStorageInstance.getStorageDescriptor();
|
||||||
|
|
||||||
s3Client = new S3Client({
|
s3Client = new S3Client({
|
||||||
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
||||||
@@ -101,8 +101,8 @@ tap.test('should delete the bucket', async () => {
|
|||||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.test('should stop the S3 server', async () => {
|
tap.test('should stop the storage server', async () => {
|
||||||
await testSmarts3Instance.stop();
|
await testSmartStorageInstance.stop();
|
||||||
});
|
});
|
||||||
|
|
||||||
export default tap.start();
|
export default tap.start();
|
||||||
|
|||||||
335
test/test.policy-actions.node.ts
Normal file
335
test/test.policy-actions.node.ts
Normal file
@@ -0,0 +1,335 @@
|
|||||||
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteBucketCommand,
|
||||||
|
ListBucketsCommand,
|
||||||
|
ListObjectsV2Command,
|
||||||
|
PutObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
CopyObjectCommand,
|
||||||
|
HeadBucketCommand,
|
||||||
|
PutBucketPolicyCommand,
|
||||||
|
GetBucketPolicyCommand,
|
||||||
|
DeleteBucketPolicyCommand,
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
|
let authClient: S3Client;
|
||||||
|
|
||||||
|
const TEST_PORT = 3347;
|
||||||
|
const ACCESS_KEY = 'TESTAKID';
|
||||||
|
const SECRET_KEY = 'TESTSECRETKEY123';
|
||||||
|
const BUCKET = 'actions-bucket';
|
||||||
|
const BASE_URL = `http://localhost:${TEST_PORT}`;
|
||||||
|
|
||||||
|
async function putPolicy(statements: any[]) {
|
||||||
|
await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({ Version: '2012-10-17', Statement: statements }),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clearPolicy() {
|
||||||
|
await authClient.send(new DeleteBucketPolicyCommand({ Bucket: BUCKET }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function denyStatement(action: string) {
|
||||||
|
return {
|
||||||
|
Sid: `Deny_${action.replace(':', '_')}`,
|
||||||
|
Effect: 'Deny' as const,
|
||||||
|
Principal: '*',
|
||||||
|
Action: action,
|
||||||
|
Resource: [
|
||||||
|
`arn:aws:s3:::${BUCKET}`,
|
||||||
|
`arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Server setup
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('setup: start server, create bucket, upload object', async () => {
|
||||||
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
|
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
|
||||||
|
storage: { cleanSlate: true },
|
||||||
|
auth: {
|
||||||
|
enabled: true,
|
||||||
|
credentials: [{ accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
authClient = new S3Client({
|
||||||
|
endpoint: BASE_URL,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: { accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY },
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await authClient.send(new CreateBucketCommand({ Bucket: BUCKET }));
|
||||||
|
await authClient.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Key: 'obj.txt',
|
||||||
|
Body: 'test content for actions',
|
||||||
|
ContentType: 'text/plain',
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Per-action deny enforcement
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Deny s3:ListBucket → authenticated ListObjects fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:ListBucket')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new ListObjectsV2Command({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:CreateBucket → authenticated CreateBucket on new bucket fails', async () => {
|
||||||
|
// We need to create a policy on the target bucket, but the target doesn't exist yet.
|
||||||
|
// Instead, we use a different approach: deny on existing bucket and test HeadBucket works
|
||||||
|
// but for CreateBucket, use fetch to target a new bucket name with the deny check.
|
||||||
|
// Actually, CreateBucket has no bucket policy to evaluate against (the bucket doesn't exist yet).
|
||||||
|
// The deny would need to be on the bucket being created.
|
||||||
|
// Since the bucket doesn't exist, there's no policy to load — so CreateBucket can't be denied via policy.
|
||||||
|
// This is expected AWS behavior. Skip this test and note it.
|
||||||
|
|
||||||
|
// Verify CreateBucket still works (no policy can deny it since bucket doesn't exist yet)
|
||||||
|
await authClient.send(new CreateBucketCommand({ Bucket: 'new-test-bucket' }));
|
||||||
|
await authClient.send(new DeleteBucketCommand({ Bucket: 'new-test-bucket' }));
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:DeleteBucket → authenticated DeleteBucket fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:DeleteBucket')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:GetObject → authenticated GetObject fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:GetObject')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new GetObjectCommand({ Bucket: BUCKET, Key: 'obj.txt' }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:PutObject → authenticated PutObject fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:PutObject')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Key: 'new-obj.txt',
|
||||||
|
Body: 'should fail',
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:DeleteObject → authenticated DeleteObject fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:DeleteObject')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'obj.txt' }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:PutObject → authenticated CopyObject fails (maps to s3:PutObject)', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:PutObject')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new CopyObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Key: 'obj-copy.txt',
|
||||||
|
CopySource: `${BUCKET}/obj.txt`,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:GetBucketPolicy → authenticated GetBucketPolicy fails', async () => {
|
||||||
|
// First put a policy that denies GetBucketPolicy
|
||||||
|
// We need to be careful: put the deny policy, then try to get it
|
||||||
|
await putPolicy([denyStatement('s3:GetBucketPolicy')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new GetBucketPolicyCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
// Clear using direct delete (which isn't denied)
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:PutBucketPolicy → authenticated PutBucketPolicy fails (for second policy)', async () => {
|
||||||
|
// First put a policy that denies PutBucketPolicy
|
||||||
|
await putPolicy([denyStatement('s3:PutBucketPolicy')]);
|
||||||
|
|
||||||
|
// Now try to put another policy — should fail
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: 'SomeOtherPolicy',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Deny s3:DeleteBucketPolicy → authenticated DeleteBucketPolicy fails', async () => {
|
||||||
|
await putPolicy([denyStatement('s3:DeleteBucketPolicy')]);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
authClient.send(new DeleteBucketPolicyCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
|
||||||
|
// We need another way to clean up — use fetch with auth to bypass? No, the deny is on all principals.
|
||||||
|
// Actually, we can't clear the policy via SDK since delete is denied.
|
||||||
|
// The server still denies it. We need to stop and restart or use a different mechanism.
|
||||||
|
// For test cleanup, just stop the server at end and it will be wiped with cleanSlate on next start.
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Recovery: remove deny policy → authenticated operations resume working', async () => {
|
||||||
|
// The previous test left a deny policy on DeleteBucketPolicy.
|
||||||
|
// But we can work around it by stopping/restarting or if the deny is still in place.
|
||||||
|
// Actually, we denied s3:DeleteBucketPolicy but NOT s3:PutBucketPolicy.
|
||||||
|
// So we can overwrite the policy with an empty-ish one, then delete.
|
||||||
|
await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: 'AllowAll',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:*',
|
||||||
|
Resource: [`arn:aws:s3:::${BUCKET}`, `arn:aws:s3:::${BUCKET}/*`],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// Now all operations should work again
|
||||||
|
const getResp = await authClient.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'obj.txt' })
|
||||||
|
);
|
||||||
|
expect(getResp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
|
||||||
|
const listResp = await authClient.send(
|
||||||
|
new ListObjectsV2Command({ Bucket: BUCKET })
|
||||||
|
);
|
||||||
|
expect(listResp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Special cases
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('ListAllMyBuckets always requires auth → anonymous fetch to / returns 403', async () => {
|
||||||
|
const resp = await fetch(`${BASE_URL}/`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Auth disabled mode → anonymous full access works', async () => {
|
||||||
|
// Start a second server with auth disabled
|
||||||
|
const noAuthInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
|
server: { port: 3348, silent: true, region: 'us-east-1' },
|
||||||
|
storage: { cleanSlate: true },
|
||||||
|
auth: { enabled: false, credentials: [] },
|
||||||
|
});
|
||||||
|
|
||||||
|
// Anonymous operations should all work
|
||||||
|
const listResp = await fetch('http://localhost:3348/');
|
||||||
|
expect(listResp.status).toEqual(200);
|
||||||
|
|
||||||
|
// Create bucket via fetch
|
||||||
|
const createResp = await fetch('http://localhost:3348/anon-bucket', { method: 'PUT' });
|
||||||
|
expect(createResp.status).toEqual(200);
|
||||||
|
|
||||||
|
// Put object
|
||||||
|
const putResp = await fetch('http://localhost:3348/anon-bucket/file.txt', {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'hello anon',
|
||||||
|
});
|
||||||
|
expect(putResp.status).toEqual(200);
|
||||||
|
|
||||||
|
// Get object
|
||||||
|
const getResp = await fetch('http://localhost:3348/anon-bucket/file.txt');
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
const text = await getResp.text();
|
||||||
|
expect(text).toEqual('hello anon');
|
||||||
|
|
||||||
|
// Delete object
|
||||||
|
const delObjResp = await fetch('http://localhost:3348/anon-bucket/file.txt', { method: 'DELETE' });
|
||||||
|
expect(delObjResp.status).toEqual(204);
|
||||||
|
|
||||||
|
// Delete bucket
|
||||||
|
const delBucketResp = await fetch('http://localhost:3348/anon-bucket', { method: 'DELETE' });
|
||||||
|
expect(delBucketResp.status).toEqual(204);
|
||||||
|
|
||||||
|
await noAuthInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Teardown
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('teardown: clean up and stop server', async () => {
|
||||||
|
// Clean up any remaining objects
|
||||||
|
try {
|
||||||
|
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'obj.txt' }));
|
||||||
|
} catch {
|
||||||
|
// May already be deleted
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
|
||||||
|
} catch {
|
||||||
|
// May already be deleted
|
||||||
|
}
|
||||||
|
await testSmartStorageInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
252
test/test.policy-crud.node.ts
Normal file
252
test/test.policy-crud.node.ts
Normal file
@@ -0,0 +1,252 @@
|
|||||||
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteBucketCommand,
|
||||||
|
PutBucketPolicyCommand,
|
||||||
|
GetBucketPolicyCommand,
|
||||||
|
DeleteBucketPolicyCommand,
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
|
let authClient: S3Client;
|
||||||
|
|
||||||
|
const TEST_PORT = 3345;
|
||||||
|
const ACCESS_KEY = 'TESTAKID';
|
||||||
|
const SECRET_KEY = 'TESTSECRETKEY123';
|
||||||
|
const BUCKET = 'policy-crud-bucket';
|
||||||
|
|
||||||
|
function makePolicy(statements: any[]) {
|
||||||
|
return JSON.stringify({ Version: '2012-10-17', Statement: statements });
|
||||||
|
}
|
||||||
|
|
||||||
|
const validStatement = {
|
||||||
|
Sid: 'Test1',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject'],
|
||||||
|
Resource: [`arn:aws:s3:::${BUCKET}/*`],
|
||||||
|
};
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Server setup
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('setup: start storage server with auth enabled', async () => {
|
||||||
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
|
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
|
||||||
|
storage: { cleanSlate: true },
|
||||||
|
auth: {
|
||||||
|
enabled: true,
|
||||||
|
credentials: [{ accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
authClient = new S3Client({
|
||||||
|
endpoint: `http://localhost:${TEST_PORT}`,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: { accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY },
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('setup: create bucket', async () => {
|
||||||
|
await authClient.send(new CreateBucketCommand({ Bucket: BUCKET }));
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// CRUD tests
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('GET policy on bucket with no policy → throws (NoSuchBucketPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(new GetBucketPolicyCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT valid policy → 204', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: makePolicy([validStatement]),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('GET policy back → returns matching JSON', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new GetBucketPolicyCommand({ Bucket: BUCKET })
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
const policy = JSON.parse(response.Policy!);
|
||||||
|
expect(policy.Version).toEqual('2012-10-17');
|
||||||
|
expect(policy.Statement[0].Sid).toEqual('Test1');
|
||||||
|
expect(policy.Statement[0].Effect).toEqual('Allow');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT updated policy (overwrite) → 204, GET returns new version', async () => {
|
||||||
|
const updatedStatement = {
|
||||||
|
Sid: 'Updated',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:DeleteObject'],
|
||||||
|
Resource: [`arn:aws:s3:::${BUCKET}/*`],
|
||||||
|
};
|
||||||
|
|
||||||
|
const putResp = await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: makePolicy([updatedStatement]),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
expect(putResp.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
|
||||||
|
const getResp = await authClient.send(
|
||||||
|
new GetBucketPolicyCommand({ Bucket: BUCKET })
|
||||||
|
);
|
||||||
|
const policy = JSON.parse(getResp.Policy!);
|
||||||
|
expect(policy.Statement[0].Sid).toEqual('Updated');
|
||||||
|
expect(policy.Statement[0].Effect).toEqual('Deny');
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('DELETE policy → 204', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new DeleteBucketPolicyCommand({ Bucket: BUCKET })
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('DELETE policy again (idempotent) → 204', async () => {
|
||||||
|
const response = await authClient.send(
|
||||||
|
new DeleteBucketPolicyCommand({ Bucket: BUCKET })
|
||||||
|
);
|
||||||
|
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('GET policy after delete → throws', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(new GetBucketPolicyCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT policy on non-existent bucket → throws (NoSuchBucket)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: 'nonexistent-bucket-xyz',
|
||||||
|
Policy: makePolicy([validStatement]),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT invalid JSON → throws (MalformedPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: '{not valid json!!!',
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT policy with wrong version → throws (MalformedPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({
|
||||||
|
Version: '2023-01-01',
|
||||||
|
Statement: [validStatement],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT policy with empty statements array → throws (MalformedPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({
|
||||||
|
Version: '2012-10-17',
|
||||||
|
Statement: [],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT policy with action missing s3: prefix → throws (MalformedPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: makePolicy([
|
||||||
|
{
|
||||||
|
Sid: 'BadAction',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['GetObject'],
|
||||||
|
Resource: [`arn:aws:s3:::${BUCKET}/*`],
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('PUT policy with resource missing arn:aws:s3::: prefix → throws (MalformedPolicy)', async () => {
|
||||||
|
await expect(
|
||||||
|
authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: makePolicy([
|
||||||
|
{
|
||||||
|
Sid: 'BadResource',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject'],
|
||||||
|
Resource: ['policy-crud-bucket/*'],
|
||||||
|
},
|
||||||
|
]),
|
||||||
|
})
|
||||||
|
)
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Bucket deletion cleans up associated policy', async () => {
|
||||||
|
// PUT a policy
|
||||||
|
await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: makePolicy([validStatement]),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// Delete the bucket
|
||||||
|
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
|
||||||
|
|
||||||
|
// Re-create the bucket
|
||||||
|
await authClient.send(new CreateBucketCommand({ Bucket: BUCKET }));
|
||||||
|
|
||||||
|
// GET policy should now be gone
|
||||||
|
await expect(
|
||||||
|
authClient.send(new GetBucketPolicyCommand({ Bucket: BUCKET }))
|
||||||
|
).rejects.toThrow();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Teardown
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('teardown: delete bucket and stop server', async () => {
|
||||||
|
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
|
||||||
|
await testSmartStorageInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
517
test/test.policy-eval.node.ts
Normal file
517
test/test.policy-eval.node.ts
Normal file
@@ -0,0 +1,517 @@
|
|||||||
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
|
import {
|
||||||
|
S3Client,
|
||||||
|
CreateBucketCommand,
|
||||||
|
DeleteBucketCommand,
|
||||||
|
PutObjectCommand,
|
||||||
|
GetObjectCommand,
|
||||||
|
DeleteObjectCommand,
|
||||||
|
PutBucketPolicyCommand,
|
||||||
|
DeleteBucketPolicyCommand,
|
||||||
|
} from '@aws-sdk/client-s3';
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
|
let authClient: S3Client;
|
||||||
|
|
||||||
|
const TEST_PORT = 3346;
|
||||||
|
const ACCESS_KEY = 'TESTAKID';
|
||||||
|
const SECRET_KEY = 'TESTSECRETKEY123';
|
||||||
|
const BUCKET = 'eval-bucket';
|
||||||
|
const BASE_URL = `http://localhost:${TEST_PORT}`;
|
||||||
|
|
||||||
|
async function streamToString(stream: Readable): Promise<string> {
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
||||||
|
stream.on('error', reject);
|
||||||
|
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function putPolicy(statements: any[]) {
|
||||||
|
await authClient.send(
|
||||||
|
new PutBucketPolicyCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Policy: JSON.stringify({ Version: '2012-10-17', Statement: statements }),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function clearPolicy() {
|
||||||
|
await authClient.send(new DeleteBucketPolicyCommand({ Bucket: BUCKET }));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Server setup
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('setup: start server, create bucket, upload object', async () => {
|
||||||
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
|
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
|
||||||
|
storage: { cleanSlate: true },
|
||||||
|
auth: {
|
||||||
|
enabled: true,
|
||||||
|
credentials: [{ accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
authClient = new S3Client({
|
||||||
|
endpoint: BASE_URL,
|
||||||
|
region: 'us-east-1',
|
||||||
|
credentials: { accessKeyId: ACCESS_KEY, secretAccessKey: SECRET_KEY },
|
||||||
|
forcePathStyle: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await authClient.send(new CreateBucketCommand({ Bucket: BUCKET }));
|
||||||
|
await authClient.send(
|
||||||
|
new PutObjectCommand({
|
||||||
|
Bucket: BUCKET,
|
||||||
|
Key: 'test-obj.txt',
|
||||||
|
Body: 'hello policy eval',
|
||||||
|
ContentType: 'text/plain',
|
||||||
|
})
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Principal matching
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Principal: "*" → anonymous fetch GET succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'PrincipalWildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
const text = await resp.text();
|
||||||
|
expect(text).toEqual('hello policy eval');
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Principal: {"AWS": "*"} → anonymous GET fails, authenticated GET succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AwsWildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: { AWS: '*' },
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Anonymous → no identity → Principal AWS:* doesn't match anonymous → NoOpinion → denied
|
||||||
|
const anonResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(anonResp.status).toEqual(403);
|
||||||
|
|
||||||
|
// Authenticated → has identity → Principal AWS:* matches → Allow
|
||||||
|
const authResp = await authClient.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' })
|
||||||
|
);
|
||||||
|
expect(authResp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Principal: {"AWS": "arn:aws:iam::TESTAKID"} → authenticated GET succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'SpecificPrincipal',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: { AWS: `arn:aws:iam::${ACCESS_KEY}` },
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await authClient.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' })
|
||||||
|
);
|
||||||
|
expect(resp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Principal: {"AWS": "arn:aws:iam::WRONGKEY"} → authenticated GET still succeeds (default allow)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'WrongPrincipal',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: { AWS: 'arn:aws:iam::WRONGKEY' },
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Principal doesn't match our key → NoOpinion → default allow for authenticated
|
||||||
|
const resp = await authClient.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' })
|
||||||
|
);
|
||||||
|
expect(resp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Action matching
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Action: "s3:*" → anonymous can GET and PUT (wildcard matches all)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'S3Wildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:*',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const getResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const putResp = await fetch(`${BASE_URL}/${BUCKET}/anon-wildcard.txt`, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'wildcard put',
|
||||||
|
});
|
||||||
|
expect(putResp.status).toEqual(200);
|
||||||
|
|
||||||
|
// Clean up the object we created
|
||||||
|
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'anon-wildcard.txt' }));
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Action: "*" → global wildcard matches all actions', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'GlobalWildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: '*',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const getResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const putResp = await fetch(`${BASE_URL}/${BUCKET}/anon-global.txt`, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'global wildcard',
|
||||||
|
});
|
||||||
|
expect(putResp.status).toEqual(200);
|
||||||
|
|
||||||
|
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'anon-global.txt' }));
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Action: "s3:Get*" → anonymous can GET but not PUT (prefix wildcard)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'PrefixWildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:Get*',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const getResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const putResp = await fetch(`${BASE_URL}/${BUCKET}/anon-prefix.txt`, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'should fail',
|
||||||
|
});
|
||||||
|
expect(putResp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Action: ["s3:GetObject", "s3:PutObject"] → anonymous can GET and PUT but not DELETE', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'MultiAction',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: ['s3:GetObject', 's3:PutObject'],
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const getResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const putResp = await fetch(`${BASE_URL}/${BUCKET}/anon-multi.txt`, {
|
||||||
|
method: 'PUT',
|
||||||
|
body: 'multi action',
|
||||||
|
});
|
||||||
|
expect(putResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const delResp = await fetch(`${BASE_URL}/${BUCKET}/anon-multi.txt`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
});
|
||||||
|
expect(delResp.status).toEqual(403);
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'anon-multi.txt' }));
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Resource ARN matching
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Resource: "arn:aws:s3:::eval-bucket/*" → anonymous GET of object succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'ResourceWildcard',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Resource: exact key → anonymous GET of that key succeeds, other key fails', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'ExactResource',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/test-obj.txt`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const goodResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(goodResp.status).toEqual(200);
|
||||||
|
|
||||||
|
// Other key → resource doesn't match → NoOpinion → denied for anonymous
|
||||||
|
const badResp = await fetch(`${BASE_URL}/${BUCKET}/nonexistent.txt`);
|
||||||
|
expect(badResp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Resource: wrong bucket ARN → NoOpinion → anonymous GET denied', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'WrongBucket',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: 'arn:aws:s3:::other-bucket/*',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Resource: "*" → matches everything, anonymous GET succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'StarResource',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: '*',
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Deny-over-Allow priority
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Allow + Deny same action → anonymous GET denied', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowGet',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: 'DenyGet',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Allow s3:* + Deny s3:DeleteObject → anonymous GET succeeds, DELETE denied', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowAll',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:*',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: 'DenyDelete',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:DeleteObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const getResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(getResp.status).toEqual(200);
|
||||||
|
|
||||||
|
const delResp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`, { method: 'DELETE' });
|
||||||
|
expect(delResp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Statement order does not matter: Deny first, Allow second → still denied', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'DenyFirst',
|
||||||
|
Effect: 'Deny',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: 'AllowSecond',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// NoOpinion fallback
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('NoOpinion: policy allows PutObject only → authenticated GET falls through (default allow)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowPutOnly',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:PutObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Authenticated → NoOpinion → default allow
|
||||||
|
const resp = await authClient.send(
|
||||||
|
new GetObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' })
|
||||||
|
);
|
||||||
|
expect(resp.$metadata.httpStatusCode).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('NoOpinion: same policy → anonymous GET falls through → default deny (403)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowPutOnly',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:PutObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Anonymous → NoOpinion for GetObject → default deny
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`);
|
||||||
|
expect(resp.status).toEqual(403);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// IAM action mapping
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('Policy allows s3:GetObject → anonymous HEAD object succeeds (HeadObject maps to s3:GetObject)', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowGet',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:GetObject',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}/*`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}/test-obj.txt`, { method: 'HEAD' });
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Policy allows s3:ListBucket → anonymous HEAD bucket succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowList',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:ListBucket',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}`, { method: 'HEAD' });
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
tap.test('Policy allows s3:ListBucket → anonymous GET bucket (list objects) succeeds', async () => {
|
||||||
|
await putPolicy([
|
||||||
|
{
|
||||||
|
Sid: 'AllowList',
|
||||||
|
Effect: 'Allow',
|
||||||
|
Principal: '*',
|
||||||
|
Action: 's3:ListBucket',
|
||||||
|
Resource: `arn:aws:s3:::${BUCKET}`,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const resp = await fetch(`${BASE_URL}/${BUCKET}`);
|
||||||
|
expect(resp.status).toEqual(200);
|
||||||
|
const text = await resp.text();
|
||||||
|
expect(text).toInclude('ListBucketResult');
|
||||||
|
await clearPolicy();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ============================
|
||||||
|
// Teardown
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
tap.test('teardown: clean up and stop server', async () => {
|
||||||
|
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' }));
|
||||||
|
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
|
||||||
|
await testSmartStorageInstance.stop();
|
||||||
|
});
|
||||||
|
|
||||||
|
export default tap.start();
|
||||||
12
test/test.ts
12
test/test.ts
@@ -1,12 +1,12 @@
|
|||||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
|
|
||||||
import * as smarts3 from '../ts/index.js';
|
import * as smartstorage from '../ts/index.js';
|
||||||
|
|
||||||
let testSmarts3Instance: smarts3.Smarts3;
|
let testSmartStorageInstance: smartstorage.SmartStorage;
|
||||||
|
|
||||||
tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
|
tap.test('should create a smartstorage instance and run it', async (toolsArg) => {
|
||||||
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
|
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
|
||||||
server: {
|
server: {
|
||||||
port: 3333,
|
port: 3333,
|
||||||
},
|
},
|
||||||
@@ -20,7 +20,7 @@ tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
|
|||||||
|
|
||||||
tap.test('should be able to access buckets', async () => {
|
tap.test('should be able to access buckets', async () => {
|
||||||
const smartbucketInstance = new plugins.smartbucket.SmartBucket(
|
const smartbucketInstance = new plugins.smartbucket.SmartBucket(
|
||||||
await testSmarts3Instance.getS3Descriptor(),
|
await testSmartStorageInstance.getStorageDescriptor(),
|
||||||
);
|
);
|
||||||
const bucket = await smartbucketInstance.createBucket('testbucket');
|
const bucket = await smartbucketInstance.createBucket('testbucket');
|
||||||
const baseDirectory = await bucket.getBaseDirectory();
|
const baseDirectory = await bucket.getBaseDirectory();
|
||||||
@@ -31,7 +31,7 @@ tap.test('should be able to access buckets', async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
tap.test('should stop the instance', async () => {
|
tap.test('should stop the instance', async () => {
|
||||||
await testSmarts3Instance.stop();
|
await testSmartStorageInstance.stop();
|
||||||
});
|
});
|
||||||
|
|
||||||
tap.start();
|
tap.start();
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
* autocreated commitinfo by @push.rocks/commitinfo
|
* autocreated commitinfo by @push.rocks/commitinfo
|
||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smarts3',
|
name: '@push.rocks/smartstorage',
|
||||||
version: '5.1.1',
|
version: '6.0.0',
|
||||||
description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.'
|
description: 'A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.'
|
||||||
}
|
}
|
||||||
|
|||||||
56
ts/index.ts
56
ts/index.ts
@@ -58,6 +58,7 @@ export interface IServerConfig {
|
|||||||
port?: number;
|
port?: number;
|
||||||
address?: string;
|
address?: string;
|
||||||
silent?: boolean;
|
silent?: boolean;
|
||||||
|
region?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -69,9 +70,9 @@ export interface IStorageConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Complete smarts3 configuration
|
* Complete smartstorage configuration
|
||||||
*/
|
*/
|
||||||
export interface ISmarts3Config {
|
export interface ISmartStorageConfig {
|
||||||
server?: IServerConfig;
|
server?: IServerConfig;
|
||||||
storage?: IStorageConfig;
|
storage?: IStorageConfig;
|
||||||
auth?: IAuthConfig;
|
auth?: IAuthConfig;
|
||||||
@@ -84,11 +85,12 @@ export interface ISmarts3Config {
|
|||||||
/**
|
/**
|
||||||
* Default configuration values
|
* Default configuration values
|
||||||
*/
|
*/
|
||||||
const DEFAULT_CONFIG: ISmarts3Config = {
|
const DEFAULT_CONFIG: ISmartStorageConfig = {
|
||||||
server: {
|
server: {
|
||||||
port: 3000,
|
port: 3000,
|
||||||
address: '0.0.0.0',
|
address: '0.0.0.0',
|
||||||
silent: false,
|
silent: false,
|
||||||
|
region: 'us-east-1',
|
||||||
},
|
},
|
||||||
storage: {
|
storage: {
|
||||||
directory: paths.bucketsDir,
|
directory: paths.bucketsDir,
|
||||||
@@ -98,8 +100,8 @@ const DEFAULT_CONFIG: ISmarts3Config = {
|
|||||||
enabled: false,
|
enabled: false,
|
||||||
credentials: [
|
credentials: [
|
||||||
{
|
{
|
||||||
accessKeyId: 'S3RVER',
|
accessKeyId: 'STORAGE',
|
||||||
secretAccessKey: 'S3RVER',
|
secretAccessKey: 'STORAGE',
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -131,7 +133,7 @@ const DEFAULT_CONFIG: ISmarts3Config = {
|
|||||||
/**
|
/**
|
||||||
* Merge user config with defaults (deep merge)
|
* Merge user config with defaults (deep merge)
|
||||||
*/
|
*/
|
||||||
function mergeConfig(userConfig: ISmarts3Config): Required<ISmarts3Config> {
|
function mergeConfig(userConfig: ISmartStorageConfig): Required<ISmartStorageConfig> {
|
||||||
return {
|
return {
|
||||||
server: {
|
server: {
|
||||||
...DEFAULT_CONFIG.server!,
|
...DEFAULT_CONFIG.server!,
|
||||||
@@ -167,35 +169,35 @@ function mergeConfig(userConfig: ISmarts3Config): Required<ISmarts3Config> {
|
|||||||
/**
|
/**
|
||||||
* IPC command type map for RustBridge
|
* IPC command type map for RustBridge
|
||||||
*/
|
*/
|
||||||
type TRustS3Commands = {
|
type TRustStorageCommands = {
|
||||||
start: { params: { config: Required<ISmarts3Config> }; result: {} };
|
start: { params: { config: Required<ISmartStorageConfig> }; result: {} };
|
||||||
stop: { params: {}; result: {} };
|
stop: { params: {}; result: {} };
|
||||||
createBucket: { params: { name: string }; result: {} };
|
createBucket: { params: { name: string }; result: {} };
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Main Smarts3 class - production-ready S3-compatible server
|
* Main SmartStorage class - production-ready S3-compatible storage server
|
||||||
*/
|
*/
|
||||||
export class Smarts3 {
|
export class SmartStorage {
|
||||||
// STATIC
|
// STATIC
|
||||||
public static async createAndStart(configArg: ISmarts3Config = {}) {
|
public static async createAndStart(configArg: ISmartStorageConfig = {}) {
|
||||||
const smartS3Instance = new Smarts3(configArg);
|
const smartStorageInstance = new SmartStorage(configArg);
|
||||||
await smartS3Instance.start();
|
await smartStorageInstance.start();
|
||||||
return smartS3Instance;
|
return smartStorageInstance;
|
||||||
}
|
}
|
||||||
|
|
||||||
// INSTANCE
|
// INSTANCE
|
||||||
public config: Required<ISmarts3Config>;
|
public config: Required<ISmartStorageConfig>;
|
||||||
private bridge: InstanceType<typeof plugins.RustBridge<TRustS3Commands>>;
|
private bridge: InstanceType<typeof plugins.RustBridge<TRustStorageCommands>>;
|
||||||
|
|
||||||
constructor(configArg: ISmarts3Config = {}) {
|
constructor(configArg: ISmartStorageConfig = {}) {
|
||||||
this.config = mergeConfig(configArg);
|
this.config = mergeConfig(configArg);
|
||||||
this.bridge = new plugins.RustBridge<TRustS3Commands>({
|
this.bridge = new plugins.RustBridge<TRustStorageCommands>({
|
||||||
binaryName: 'rusts3',
|
binaryName: 'ruststorage',
|
||||||
localPaths: [
|
localPaths: [
|
||||||
plugins.path.join(paths.packageDir, 'dist_rust', 'rusts3'),
|
plugins.path.join(paths.packageDir, 'dist_rust', 'ruststorage'),
|
||||||
plugins.path.join(paths.packageDir, 'rust', 'target', 'release', 'rusts3'),
|
plugins.path.join(paths.packageDir, 'rust', 'target', 'release', 'ruststorage'),
|
||||||
plugins.path.join(paths.packageDir, 'rust', 'target', 'debug', 'rusts3'),
|
plugins.path.join(paths.packageDir, 'rust', 'target', 'debug', 'ruststorage'),
|
||||||
],
|
],
|
||||||
readyTimeoutMs: 30000,
|
readyTimeoutMs: 30000,
|
||||||
requestTimeoutMs: 300000,
|
requestTimeoutMs: 300000,
|
||||||
@@ -205,21 +207,21 @@ export class Smarts3 {
|
|||||||
public async start() {
|
public async start() {
|
||||||
const spawned = await this.bridge.spawn();
|
const spawned = await this.bridge.spawn();
|
||||||
if (!spawned) {
|
if (!spawned) {
|
||||||
throw new Error('Failed to spawn rusts3 binary. Make sure it is compiled (pnpm build).');
|
throw new Error('Failed to spawn ruststorage binary. Make sure it is compiled (pnpm build).');
|
||||||
}
|
}
|
||||||
await this.bridge.sendCommand('start', { config: this.config });
|
await this.bridge.sendCommand('start', { config: this.config });
|
||||||
|
|
||||||
if (!this.config.server.silent) {
|
if (!this.config.server.silent) {
|
||||||
console.log('s3 server is running');
|
console.log('storage server is running');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getS3Descriptor(
|
public async getStorageDescriptor(
|
||||||
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
|
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
|
||||||
): Promise<plugins.tsclass.storage.IS3Descriptor> {
|
): Promise<plugins.tsclass.storage.IS3Descriptor> {
|
||||||
const cred = this.config.auth.credentials[0] || {
|
const cred = this.config.auth.credentials[0] || {
|
||||||
accessKeyId: 'S3RVER',
|
accessKeyId: 'STORAGE',
|
||||||
secretAccessKey: 'S3RVER',
|
secretAccessKey: 'STORAGE',
|
||||||
};
|
};
|
||||||
|
|
||||||
const descriptor: plugins.tsclass.storage.IS3Descriptor = {
|
const descriptor: plugins.tsclass.storage.IS3Descriptor = {
|
||||||
|
|||||||
Reference in New Issue
Block a user