Compare commits

...

3 Commits
v5.2.0 ... main

Author SHA1 Message Date
bba0855218 BREAKING CHANGE(core): rebrand from smarts3 to smartstorage
Some checks failed
Default (tags) / security (push) Successful in 43s
Default (tags) / test (push) Failing after 26s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
- Package renamed from @push.rocks/smarts3 to @push.rocks/smartstorage
- Class: Smarts3 → SmartStorage, Interface: ISmarts3Config → ISmartStorageConfig
- Method: getS3Descriptor → getStorageDescriptor
- Rust binary: rusts3 → ruststorage
- Rust types: S3Error→StorageError, S3Action→StorageAction, S3Config→SmartStorageConfig, S3Server→StorageServer
- On-disk file extension: ._S3_object → ._storage_object
- Default credentials: S3RVER → STORAGE
- All internal S3 branding removed; AWS S3 protocol compatibility fully maintained
2026-03-14 15:20:30 +00:00
d437ffc226 v5.3.0
Some checks failed
Default (tags) / security (push) Successful in 37s
Default (tags) / test (push) Failing after 26s
Default (tags) / release (push) Has been skipped
Default (tags) / metadata (push) Has been skipped
2026-02-17 16:50:04 +00:00
e36758f183 feat(auth): add AWS SigV4 authentication and bucket policy support 2026-02-17 16:50:04 +00:00
26 changed files with 412 additions and 410 deletions

View File

@@ -1,5 +1,26 @@
# Changelog
## 2026-03-14 - 6.0.0 - BREAKING CHANGE(core)
Rebrand from smarts3 to smartstorage
- Package renamed from @push.rocks/smarts3 to @push.rocks/smartstorage
- Class renamed from Smarts3 to SmartStorage (no backward-compatible re-export)
- Interface renamed from ISmarts3Config to ISmartStorageConfig
- Method renamed from getS3Descriptor to getStorageDescriptor
- Rust binary renamed from rusts3 to ruststorage
- Rust types renamed: S3Error→StorageError, S3Action→StorageAction, S3Config→SmartStorageConfig, S3Server→StorageServer
- On-disk file extension changed from ._S3_object to ._storage_object (BREAKING for existing stored data)
- Default credentials changed from S3RVER to STORAGE
- All internal S3 branding removed; AWS S3 protocol compatibility (IAM actions, ARNs, SigV4) fully maintained
## 2026-02-17 - 5.3.0 - feat(auth)
add AWS SigV4 authentication and bucket policy support
- Implement AWS SigV4 full verification (constant-time comparison, 15-minute clock skew enforcement) and expose default signing region (server.region = 'us-east-1').
- Add IAM-style bucket policy engine with Put/Get/Delete policy APIs (GetBucketPolicy/PutBucketPolicy/DeleteBucketPolicy), wildcard action/resource matching, Allow/Deny evaluation, and on-disk persistence under .policies/{bucket}.policy.json.
- Documentation and README expanded with policy usage, examples, API table entries, and notes about policy CRUD and behavior for anonymous/authenticated requests.
- Rust code refactors: simplify storage/server result structs and multipart handling (removed several unused size/key/bucket fields), remove S3Error::to_response and error_xml helpers, and other internal cleanup to support new auth/policy features.
## 2026-02-17 - 5.2.0 - feat(auth,policy)
add AWS SigV4 authentication and S3 bucket policy support

View File

@@ -10,14 +10,15 @@
"module": {
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smarts3",
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
"npmPackagename": "@push.rocks/smarts3",
"gitrepo": "smartstorage",
"description": "A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.",
"npmPackagename": "@push.rocks/smartstorage",
"license": "MIT",
"projectDomain": "push.rocks",
"keywords": [
"S3 Mock Server",
"Local S3",
"smartstorage",
"S3 Compatible",
"Local Storage Server",
"Node.js",
"TypeScript",
"Local Development",
@@ -26,8 +27,8 @@
"File Storage",
"AWS S3 Compatibility",
"Development Tool",
"S3 Endpoint",
"S3 Simulation",
"Storage Endpoint",
"Storage Simulation",
"Bucket Management",
"File Upload",
"CI/CD Integration",

View File

@@ -1,8 +1,8 @@
{
"name": "@push.rocks/smarts3",
"version": "5.2.0",
"name": "@push.rocks/smartstorage",
"version": "6.0.0",
"private": false,
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
"description": "A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@@ -45,8 +45,9 @@
"@tsclass/tsclass": "^9.3.0"
},
"keywords": [
"S3 Mock Server",
"Local S3",
"smartstorage",
"S3 Compatible",
"Local Storage Server",
"Node.js",
"TypeScript",
"Local Development",
@@ -55,20 +56,20 @@
"File Storage",
"AWS S3 Compatibility",
"Development Tool",
"S3 Endpoint",
"S3 Simulation",
"Storage Endpoint",
"Storage Simulation",
"Bucket Management",
"File Upload",
"CI/CD Integration",
"Developer Onboarding"
],
"homepage": "https://code.foss.global/push.rocks/smarts3#readme",
"homepage": "https://code.foss.global/push.rocks/smartstorage#readme",
"repository": {
"type": "git",
"url": "https://code.foss.global/push.rocks/smarts3.git"
"url": "ssh://git@code.foss.global:29419/push.rocks/smartstorage.git"
},
"bugs": {
"url": "https://code.foss.global/push.rocks/smarts3/issues"
"url": "https://code.foss.global/push.rocks/smartstorage/issues"
},
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748",
"pnpm": {

View File

@@ -1,6 +1,6 @@
# Production-Readiness Plan for smarts3
# Production-Readiness Plan for smartstorage
**Goal:** Make smarts3 production-ready as a MinIO alternative for use cases where:
**Goal:** Make smartstorage production-ready as a MinIO alternative for use cases where:
- Running MinIO is out of scope
- You have a program written for S3 and want to use the local filesystem
- You need a lightweight, zero-dependency S3-compatible server
@@ -31,7 +31,7 @@
### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY**
**Why:** Essential for uploading files >5MB efficiently. Without this, smarts3 can't handle real-world production workloads.
**Why:** Essential for uploading files >5MB efficiently. Without this, smartstorage can't handle real-world production workloads.
**Implementation Required:**
- `POST /:bucket/:key?uploads` - CreateMultipartUpload
@@ -46,13 +46,13 @@
**Files to Create/Modify:**
- `ts/controllers/multipart.controller.ts` (new)
- `ts/classes/filesystem-store.ts` (add multipart methods)
- `ts/classes/smarts3-server.ts` (add multipart routes)
- `ts/classes/smartstorage-server.ts` (add multipart routes)
---
### 2. Configurable Authentication 🔐
**Why:** Currently hardcoded credentials ('S3RVER'/'S3RVER'). Production needs custom credentials.
**Why:** Currently hardcoded credentials ('STORAGE'/'STORAGE'). Production needs custom credentials.
**Implementation Required:**
- Support custom access keys and secrets via configuration
@@ -75,7 +75,7 @@ interface IAuthConfig {
**Files to Create/Modify:**
- `ts/classes/auth-middleware.ts` (new)
- `ts/classes/signature-validator.ts` (new)
- `ts/classes/smarts3-server.ts` (integrate auth middleware)
- `ts/classes/smartstorage-server.ts` (integrate auth middleware)
- `ts/index.ts` (add auth config options)
---
@@ -105,7 +105,7 @@ interface ICorsConfig {
**Files to Create/Modify:**
- `ts/classes/cors-middleware.ts` (new)
- `ts/classes/smarts3-server.ts` (integrate CORS middleware)
- `ts/classes/smartstorage-server.ts` (integrate CORS middleware)
- `ts/index.ts` (add CORS config options)
---
@@ -131,7 +131,7 @@ interface ISslConfig {
```
**Files to Create/Modify:**
- `ts/classes/smarts3-server.ts` (add HTTPS server creation)
- `ts/classes/smartstorage-server.ts` (add HTTPS server creation)
- `ts/index.ts` (add SSL config options)
---
@@ -147,7 +147,7 @@ interface ISslConfig {
- Sensible production defaults
- Example configurations for common use cases
**Configuration File Example (`smarts3.config.json`):**
**Configuration File Example (`smartstorage.config.json`):**
```json
{
"server": {
@@ -220,7 +220,7 @@ interface ISslConfig {
**Files to Create/Modify:**
- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?)
- `ts/classes/access-logger-middleware.ts` (new)
- `ts/classes/smarts3-server.ts` (replace console.log with logger)
- `ts/classes/smartstorage-server.ts` (replace console.log with logger)
- All controller files (use structured logging)
---
@@ -238,7 +238,7 @@ interface ISslConfig {
**Files to Create/Modify:**
- `ts/controllers/health.controller.ts` (new)
- `ts/classes/metrics-collector.ts` (new)
- `ts/classes/smarts3-server.ts` (add health routes)
- `ts/classes/smartstorage-server.ts` (add health routes)
---
@@ -266,7 +266,7 @@ interface ISslConfig {
**Files to Create/Modify:**
- `ts/classes/validation-middleware.ts` (new)
- `ts/utils/validators.ts` (new)
- `ts/classes/smarts3-server.ts` (integrate validation middleware)
- `ts/classes/smartstorage-server.ts` (integrate validation middleware)
---
@@ -291,7 +291,7 @@ interface ISslConfig {
- SIGTERM/SIGINT handling
**Files to Create/Modify:**
- `ts/classes/smarts3-server.ts` (add graceful shutdown logic)
- `ts/classes/smartstorage-server.ts` (add graceful shutdown logic)
- `ts/index.ts` (add signal handlers)
---
@@ -336,7 +336,7 @@ interface ISslConfig {
4. ✅ Production configuration system
5. ✅ Production logging
**Outcome:** smarts3 can handle real production workloads
**Outcome:** smartstorage can handle real production workloads
---
@@ -350,7 +350,7 @@ interface ISslConfig {
9. ✅ Graceful shutdown
10. ✅ Batch operations
**Outcome:** smarts3 is operationally mature
**Outcome:** smartstorage is operationally mature
---
@@ -363,7 +363,7 @@ interface ISslConfig {
13. ✅ Comprehensive test suite
14. ✅ Documentation updates
**Outcome:** smarts3 has broad S3 API compatibility
**Outcome:** smartstorage has broad S3 API compatibility
---
@@ -375,7 +375,7 @@ interface ISslConfig {
16. ✅ Performance optimization
17. ✅ Advanced features based on user feedback
**Outcome:** smarts3 is a complete MinIO alternative
**Outcome:** smartstorage is a complete MinIO alternative
---
@@ -392,7 +392,7 @@ interface ISslConfig {
## 🎯 Target Use Cases
**With this plan implemented, smarts3 will be a solid MinIO alternative for:**
**With this plan implemented, smartstorage will be a solid MinIO alternative for:**
**Local S3 development** - Fast, simple, no Docker required
**Testing S3 integrations** - Reliable, repeatable tests

View File

@@ -1,10 +1,10 @@
# Project Hints for smarts3
# Project Hints for smartstorage
## Current State (v6.0.0-dev)
## Current State (v6.0.0)
- **Rust-powered S3 server** via `@push.rocks/smartrust` IPC bridge
- **Rust-powered S3-compatible storage server** via `@push.rocks/smartrust` IPC bridge
- High-performance: streaming I/O, zero-copy, backpressure, range seek
- TypeScript is thin IPC wrapper; all HTTP/storage/routing in Rust binary `rusts3`
- TypeScript is thin IPC wrapper; all HTTP/storage/routing in Rust binary `ruststorage`
- Full S3 compatibility: PUT, GET, HEAD, DELETE for objects and buckets
- Multipart upload support (streaming, no OOM)
- **Real AWS SigV4 authentication** (cryptographic signature verification)
@@ -18,37 +18,37 @@
- `main.rs` - Clap CLI, management mode entry
- `config.rs` - Serde config structs matching TS interfaces (includes `region`)
- `management.rs` - IPC loop (newline-delimited JSON over stdin/stdout)
- `server.rs` - hyper 1.x HTTP server, routing, CORS, auth+policy pipeline, all S3 handlers
- `server.rs` - hyper 1.x HTTP server, routing, CORS, auth+policy pipeline, all S3-compatible handlers
- `storage.rs` - FileStore: filesystem-backed storage, multipart manager, `.policies/` dir
- `xml_response.rs` - S3 XML response builders
- `s3_error.rs` - S3 error codes with HTTP status mapping
- `xml_response.rs` - S3-compatible XML response builders
- `error.rs` - StorageError codes with HTTP status mapping
- `auth.rs` - AWS SigV4 signature verification (HMAC-SHA256, clock skew, constant-time compare)
- `action.rs` - S3Action enum + request-to-IAM-action resolver + RequestContext
- `action.rs` - StorageAction enum + request-to-IAM-action resolver + RequestContext
- `policy.rs` - BucketPolicy model, evaluation engine (Deny > Allow > NoOpinion), PolicyStore (RwLock cache + disk)
### TypeScript Bridge (`ts/`)
- `ts/index.ts` - Smarts3 class with RustBridge<TRustS3Commands>
- `ts/index.ts` - SmartStorage class with RustBridge<TRustStorageCommands>
- `ts/plugins.ts` - path, smartpath, RustBridge, tsclass
- `ts/paths.ts` - packageDir, bucketsDir defaults
### IPC Commands
| Command | Params | Action |
|---------|--------|--------|
| `start` | `{ config: ISmarts3Config }` | Init storage + HTTP server |
| `start` | `{ config: ISmartStorageConfig }` | Init storage + HTTP server |
| `stop` | `{}` | Graceful shutdown |
| `createBucket` | `{ name: string }` | Create bucket directory |
### Storage Layout (backward-compatible)
- Objects: `{root}/{bucket}/{key}._S3_object`
- Metadata: `{root}/{bucket}/{key}._S3_object.metadata.json`
- MD5: `{root}/{bucket}/{key}._S3_object.md5`
### Storage Layout
- Objects: `{root}/{bucket}/{key}._storage_object`
- Metadata: `{root}/{bucket}/{key}._storage_object.metadata.json`
- MD5: `{root}/{bucket}/{key}._storage_object.md5`
- Multipart: `{root}/.multipart/{upload_id}/part-{N}`
- Policies: `{root}/.policies/{bucket}.policy.json`
## Build
- `pnpm build` runs `tsrust && tsbuild --web --allowimplicitany`
- `tsrust` compiles Rust to `dist_rust/rusts3`
- `tsrust` compiles Rust to `dist_rust/ruststorage`
- Targets: linux_amd64, linux_arm64 (configured in npmextra.json)
## Dependencies

199
readme.md
View File

@@ -1,78 +1,81 @@
# @push.rocks/smarts3 🚀
# @push.rocks/smartstorage
A high-performance, S3-compatible local server powered by a **Rust core** with a clean TypeScript API. Drop-in replacement for AWS S3 during development and testing — no cloud, no Docker, no MinIO. Just `npm install` and go.
A high-performance, S3-compatible local storage server powered by a **Rust core** with a clean TypeScript API. Drop-in replacement for AWS S3 during development and testing — no cloud, no Docker, no MinIO. Just `npm install` and go.
## Issue Reporting and Security
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
## 🌟 Why smarts3?
## Why smartstorage?
| Feature | smarts3 | MinIO | s3rver |
|---------|---------|-------|--------|
| Feature | smartstorage | MinIO | s3rver |
|---------|-------------|-------|--------|
| Install | `pnpm add` | Docker / binary | `npm install` |
| Startup time | ~20ms | seconds | ~200ms |
| Large file uploads | Streaming, zero-copy | ✅ | ❌ OOM risk |
| Range requests | Seek-based | ✅ | ❌ Full read |
| Large file uploads | Streaming, zero-copy | Yes | OOM risk |
| Range requests | Seek-based | Yes | Full read |
| Language | Rust + TypeScript | Go | JavaScript |
| Multipart uploads | Full support | | |
| Auth | AWS v2/v4 key extraction | Full IAM | Basic |
| Multipart uploads | Full support | Yes | No |
| Auth | AWS SigV4 (full verification) | Full IAM | Basic |
| Bucket policies | IAM-style evaluation | Yes | No |
### Core Features
- **Rust-powered HTTP server** — hyper 1.x with streaming I/O, zero-copy, backpressure
- 🔄 **Full S3 API compatibility** — works with AWS SDK v3, SmartBucket, any S3 client
- 📂 **Filesystem-backed storage** — buckets map to directories, objects to files
- 📤 **Streaming multipart uploads** — large files without memory pressure
- 🎯 **Byte-range requests**`seek()` directly to the requested byte offset
- 🔐 **Authentication**AWS v2/v4 signature key extraction
- 🌐 **CORS middleware**configurable cross-origin support
- 📊 **Structured logging**tracing-based, error through debug levels
- 🧹 **Clean slate mode**wipe storage on startup for test isolation
- 🧪 **Test-first design**start/stop in milliseconds, no port conflicts
- **Rust-powered HTTP server** — hyper 1.x with streaming I/O, zero-copy, backpressure
- **Full S3-compatible API** — works with AWS SDK v3, SmartBucket, any S3 client
- **Filesystem-backed storage** — buckets map to directories, objects to files
- **Streaming multipart uploads** — large files without memory pressure
- **Byte-range requests** — `seek()` directly to the requested byte offset
- **AWS SigV4 authentication** — full signature verification with constant-time comparison and 15-min clock skew enforcement
- **Bucket policies** — IAM-style JSON policies with Allow/Deny evaluation, wildcard matching, and anonymous access support
- **CORS middleware** — configurable cross-origin support
- **Structured logging** — tracing-based, error through debug levels
- **Clean slate mode** — wipe storage on startup for test isolation
- **Test-first design** — start/stop in milliseconds, no port conflicts
## 📦 Installation
## Installation
```bash
pnpm add @push.rocks/smarts3 -D
pnpm add @push.rocks/smartstorage -D
```
> **Note:** The package ships with precompiled Rust binaries for `linux_amd64` and `linux_arm64`. No Rust toolchain needed on your machine.
## 🚀 Quick Start
## Quick Start
```typescript
import { Smarts3 } from '@push.rocks/smarts3';
import { SmartStorage } from '@push.rocks/smartstorage';
// Start a local S3 server
const s3 = await Smarts3.createAndStart({
// Start a local S3-compatible storage server
const storage = await SmartStorage.createAndStart({
server: { port: 3000 },
storage: { cleanSlate: true },
});
// Create a bucket
await s3.createBucket('my-bucket');
await storage.createBucket('my-bucket');
// Get connection details for any S3 client
const descriptor = await s3.getS3Descriptor();
// → { endpoint: 'localhost', port: 3000, accessKey: 'S3RVER', accessSecret: 'S3RVER', useSsl: false }
const descriptor = await storage.getStorageDescriptor();
// → { endpoint: 'localhost', port: 3000, accessKey: 'STORAGE', accessSecret: 'STORAGE', useSsl: false }
// When done
await s3.stop();
await storage.stop();
```
## 📖 Configuration
## Configuration
All config fields are optional — sensible defaults are applied automatically.
```typescript
import { Smarts3, ISmarts3Config } from '@push.rocks/smarts3';
import { SmartStorage, ISmartStorageConfig } from '@push.rocks/smartstorage';
const config: ISmarts3Config = {
const config: ISmartStorageConfig = {
server: {
port: 3000, // Default: 3000
address: '0.0.0.0', // Default: '0.0.0.0'
silent: false, // Default: false
region: 'us-east-1', // Default: 'us-east-1' — used for SigV4 signing
},
storage: {
directory: './my-data', // Default: .nogit/bucketsDir
@@ -110,14 +113,14 @@ const config: ISmarts3Config = {
},
};
const s3 = await Smarts3.createAndStart(config);
const storage = await SmartStorage.createAndStart(config);
```
### Common Configurations
**CI/CD testing** — silent, clean, fast:
```typescript
const s3 = await Smarts3.createAndStart({
const storage = await SmartStorage.createAndStart({
server: { port: 9999, silent: true },
storage: { cleanSlate: true },
});
@@ -125,7 +128,7 @@ const s3 = await Smarts3.createAndStart({
**Auth enabled:**
```typescript
const s3 = await Smarts3.createAndStart({
const storage = await SmartStorage.createAndStart({
auth: {
enabled: true,
credentials: [{ accessKeyId: 'test', secretAccessKey: 'test123' }],
@@ -135,7 +138,7 @@ const s3 = await Smarts3.createAndStart({
**CORS for local web dev:**
```typescript
const s3 = await Smarts3.createAndStart({
const storage = await SmartStorage.createAndStart({
cors: {
enabled: true,
allowedOrigins: ['http://localhost:5173'],
@@ -144,12 +147,12 @@ const s3 = await Smarts3.createAndStart({
});
```
## 📤 Usage with AWS SDK v3
## Usage with AWS SDK v3
```typescript
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
const descriptor = await s3.getS3Descriptor();
const descriptor = await storage.getStorageDescriptor();
const client = new S3Client({
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
@@ -158,14 +161,14 @@ const client = new S3Client({
accessKeyId: descriptor.accessKey,
secretAccessKey: descriptor.accessSecret,
},
forcePathStyle: true, // Required for path-style S3
forcePathStyle: true, // Required for path-style access
});
// Upload
await client.send(new PutObjectCommand({
Bucket: 'my-bucket',
Key: 'hello.txt',
Body: 'Hello, S3!',
Body: 'Hello, Storage!',
ContentType: 'text/plain',
}));
@@ -174,7 +177,7 @@ const { Body } = await client.send(new GetObjectCommand({
Bucket: 'my-bucket',
Key: 'hello.txt',
}));
const content = await Body.transformToString(); // "Hello, S3!"
const content = await Body.transformToString(); // "Hello, Storage!"
// Delete
await client.send(new DeleteObjectCommand({
@@ -183,12 +186,12 @@ await client.send(new DeleteObjectCommand({
}));
```
## 🪣 Usage with SmartBucket
## Usage with SmartBucket
```typescript
import { SmartBucket } from '@push.rocks/smartbucket';
const smartbucket = new SmartBucket(await s3.getS3Descriptor());
const smartbucket = new SmartBucket(await storage.getStorageDescriptor());
const bucket = await smartbucket.createBucket('my-bucket');
const dir = await bucket.getBaseDirectory();
@@ -202,9 +205,9 @@ const content = await dir.fastGet('docs/readme.txt');
const files = await dir.listFiles();
```
## 📤 Multipart Uploads
## Multipart Uploads
For files larger than 5 MB, use multipart uploads. smarts3 handles them with **streaming I/O** — parts are written directly to disk, never buffered in memory.
For files larger than 5 MB, use multipart uploads. smartstorage handles them with **streaming I/O** — parts are written directly to disk, never buffered in memory.
```typescript
import {
@@ -241,38 +244,88 @@ await client.send(new CompleteMultipartUploadCommand({
}));
```
## 🧪 Testing Integration
## Bucket Policies
smartstorage supports AWS-style bucket policies for fine-grained access control. Policies use the same IAM JSON format as real S3 — so you can develop and test your policy logic locally before deploying.
When `auth.enabled` is `true`, the auth pipeline works as follows:
1. **Authenticate** — verify the AWS SigV4 signature (anonymous requests skip this step)
2. **Authorize** — evaluate bucket policies against the request action, resource, and caller identity
3. **Default** — authenticated users get full access; anonymous requests are denied unless a policy explicitly allows them
### Setting a Bucket Policy
Use the S3 `PutBucketPolicy` API (or any S3 client that supports it):
```typescript
import { Smarts3 } from '@push.rocks/smarts3';
import { PutBucketPolicyCommand } from '@aws-sdk/client-s3';
// Allow anonymous read access to all objects in a bucket
await client.send(new PutBucketPolicyCommand({
Bucket: 'public-assets',
Policy: JSON.stringify({
Version: '2012-10-17',
Statement: [{
Sid: 'PublicRead',
Effect: 'Allow',
Principal: '*',
Action: ['s3:GetObject'],
Resource: ['arn:aws:s3:::public-assets/*'],
}],
}),
}));
```
### Policy Features
- **Effect**: `Allow` and `Deny` (explicit Deny always wins)
- **Principal**: `"*"` (everyone) or `{ "AWS": ["arn:..."] }` for specific identities
- **Action**: IAM-style actions like `s3:GetObject`, `s3:PutObject`, `s3:*`, or prefix wildcards like `s3:Get*`
- **Resource**: ARN patterns with `*` and `?` wildcards (e.g. `arn:aws:s3:::my-bucket/*`)
- **Persistence**: Policies survive server restarts — stored as JSON on disk alongside your data
### Policy CRUD Operations
| Operation | AWS SDK Command | HTTP |
|-----------|----------------|------|
| Get policy | `GetBucketPolicyCommand` | `GET /{bucket}?policy` |
| Set policy | `PutBucketPolicyCommand` | `PUT /{bucket}?policy` |
| Delete policy | `DeleteBucketPolicyCommand` | `DELETE /{bucket}?policy` |
Deleting a bucket automatically removes its associated policy.
## Testing Integration
```typescript
import { SmartStorage } from '@push.rocks/smartstorage';
import { tap, expect } from '@git.zone/tstest/tapbundle';
let s3: Smarts3;
let storage: SmartStorage;
tap.test('setup', async () => {
s3 = await Smarts3.createAndStart({
storage = await SmartStorage.createAndStart({
server: { port: 4567, silent: true },
storage: { cleanSlate: true },
});
});
tap.test('should store and retrieve objects', async () => {
await s3.createBucket('test');
await storage.createBucket('test');
// ... your test logic using AWS SDK or SmartBucket
});
tap.test('teardown', async () => {
await s3.stop();
await storage.stop();
});
export default tap.start();
```
## 🔧 API Reference
## API Reference
### `Smarts3` Class
### `SmartStorage` Class
#### `static createAndStart(config?: ISmarts3Config): Promise<Smarts3>`
#### `static createAndStart(config?: ISmartStorageConfig): Promise<SmartStorage>`
Create and start a server in one call.
@@ -286,11 +339,11 @@ Gracefully stop the server and kill the Rust process.
#### `createBucket(name: string): Promise<{ name: string }>`
Create an S3 bucket.
Create a storage bucket.
#### `getS3Descriptor(options?): Promise<IS3Descriptor>`
#### `getStorageDescriptor(options?): Promise<IS3Descriptor>`
Get connection details for S3 clients. Returns:
Get connection details for S3-compatible clients. Returns:
| Field | Type | Description |
|-------|------|-------------|
@@ -300,25 +353,26 @@ Get connection details for S3 clients. Returns:
| `accessSecret` | `string` | Secret key from first configured credential |
| `useSsl` | `boolean` | Always `false` (plain HTTP) |
## 🏗️ Architecture
## Architecture
smarts3 uses a **hybrid Rust + TypeScript** architecture:
smartstorage uses a **hybrid Rust + TypeScript** architecture:
```
┌─────────────────────────────────┐
│ Your Code (AWS SDK, etc.) │
│ ↕ HTTP (localhost:3000) │
├─────────────────────────────────┤
│ rusts3 binary (Rust)
│ ruststorage binary (Rust) │
│ ├─ hyper 1.x HTTP server │
│ ├─ S3 path-style routing │
│ ├─ Streaming storage layer │
│ ├─ Multipart manager │
│ ├─ CORS / Auth middleware
│ ├─ SigV4 auth + policy engine
│ ├─ CORS middleware │
│ └─ S3 XML response builder │
├─────────────────────────────────┤
│ TypeScript (thin IPC wrapper) │
│ ├─ Smarts3 class
│ ├─ SmartStorage class │
│ ├─ RustBridge (stdin/stdout) │
│ └─ Config & S3 descriptor │
└─────────────────────────────────┘
@@ -326,9 +380,9 @@ smarts3 uses a **hybrid Rust + TypeScript** architecture:
**Why Rust?** The TypeScript implementation had critical perf issues: OOM on multipart uploads (parts buffered in memory), double stream copying, file descriptor leaks on HEAD requests, full-file reads for range requests, and no backpressure. The Rust binary solves all of these with streaming I/O, zero-copy, and direct `seek()` for range requests.
**IPC Protocol:** TypeScript spawns the `rusts3` binary with `--management` and communicates via newline-delimited JSON over stdin/stdout. Commands: `start`, `stop`, `createBucket`.
**IPC Protocol:** TypeScript spawns the `ruststorage` binary with `--management` and communicates via newline-delimited JSON over stdin/stdout. Commands: `start`, `stop`, `createBucket`.
### S3 Operations Supported
### S3-Compatible Operations Supported
| Operation | Method | Path |
|-----------|--------|------|
@@ -347,27 +401,32 @@ smarts3 uses a **hybrid Rust + TypeScript** architecture:
| CompleteMultipartUpload | `POST /{bucket}/{key}?uploadId` | |
| AbortMultipartUpload | `DELETE /{bucket}/{key}?uploadId` | |
| ListMultipartUploads | `GET /{bucket}?uploads` | |
| GetBucketPolicy | `GET /{bucket}?policy` | |
| PutBucketPolicy | `PUT /{bucket}?policy` | |
| DeleteBucketPolicy | `DELETE /{bucket}?policy` | |
### On-Disk Format
```
{storage.directory}/
{bucket}/
{key}._S3_object # Object data
{key}._S3_object.metadata.json # Metadata (content-type, x-amz-meta-*, etc.)
{key}._S3_object.md5 # Cached MD5 hash
{key}._storage_object # Object data
{key}._storage_object.metadata.json # Metadata (content-type, x-amz-meta-*, etc.)
{key}._storage_object.md5 # Cached MD5 hash
.multipart/
{upload-id}/
metadata.json # Upload metadata (bucket, key, parts)
part-1 # Part data files
part-2
...
.policies/
{bucket}.policy.json # Bucket policy (IAM JSON format)
```
## 🔗 Related Packages
## Related Packages
- [`@push.rocks/smartbucket`](https://code.foss.global/push.rocks/smartbucket) — High-level S3 abstraction layer
- [`@push.rocks/smartrust`](https://code.foss.global/push.rocks/smartrust) — TypeScript Rust IPC bridge
- [`@push.rocks/smartbucket`](https://code.foss.global/push.rocks/smartbucket) — High-level S3-compatible abstraction layer
- [`@push.rocks/smartrust`](https://code.foss.global/push.rocks/smartrust) — TypeScript <-> Rust IPC bridge
- [`@git.zone/tsrust`](https://code.foss.global/git.zone/tsrust) — Rust cross-compilation for npm packages
## License and Legal Information

2
rust/Cargo.lock generated
View File

@@ -765,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
[[package]]
name = "rusts3"
name = "ruststorage"
version = "0.1.0"
dependencies = [
"anyhow",

View File

@@ -1,10 +1,10 @@
[package]
name = "rusts3"
name = "ruststorage"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "rusts3"
name = "ruststorage"
path = "src/main.rs"
[dependencies]

View File

@@ -2,9 +2,9 @@ use hyper::body::Incoming;
use hyper::{Method, Request};
use std::collections::HashMap;
/// S3 actions that map to IAM permission strings.
/// Storage actions that map to IAM permission strings.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum S3Action {
pub enum StorageAction {
ListAllMyBuckets,
CreateBucket,
DeleteBucket,
@@ -25,28 +25,28 @@ pub enum S3Action {
DeleteBucketPolicy,
}
impl S3Action {
impl StorageAction {
/// Return the IAM-style action string (e.g. "s3:GetObject").
pub fn iam_action(&self) -> &'static str {
match self {
S3Action::ListAllMyBuckets => "s3:ListAllMyBuckets",
S3Action::CreateBucket => "s3:CreateBucket",
S3Action::DeleteBucket => "s3:DeleteBucket",
S3Action::HeadBucket => "s3:ListBucket",
S3Action::ListBucket => "s3:ListBucket",
S3Action::GetObject => "s3:GetObject",
S3Action::HeadObject => "s3:GetObject",
S3Action::PutObject => "s3:PutObject",
S3Action::DeleteObject => "s3:DeleteObject",
S3Action::CopyObject => "s3:PutObject",
S3Action::ListBucketMultipartUploads => "s3:ListBucketMultipartUploads",
S3Action::AbortMultipartUpload => "s3:AbortMultipartUpload",
S3Action::InitiateMultipartUpload => "s3:PutObject",
S3Action::UploadPart => "s3:PutObject",
S3Action::CompleteMultipartUpload => "s3:PutObject",
S3Action::GetBucketPolicy => "s3:GetBucketPolicy",
S3Action::PutBucketPolicy => "s3:PutBucketPolicy",
S3Action::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
StorageAction::ListAllMyBuckets => "s3:ListAllMyBuckets",
StorageAction::CreateBucket => "s3:CreateBucket",
StorageAction::DeleteBucket => "s3:DeleteBucket",
StorageAction::HeadBucket => "s3:ListBucket",
StorageAction::ListBucket => "s3:ListBucket",
StorageAction::GetObject => "s3:GetObject",
StorageAction::HeadObject => "s3:GetObject",
StorageAction::PutObject => "s3:PutObject",
StorageAction::DeleteObject => "s3:DeleteObject",
StorageAction::CopyObject => "s3:PutObject",
StorageAction::ListBucketMultipartUploads => "s3:ListBucketMultipartUploads",
StorageAction::AbortMultipartUpload => "s3:AbortMultipartUpload",
StorageAction::InitiateMultipartUpload => "s3:PutObject",
StorageAction::UploadPart => "s3:PutObject",
StorageAction::CompleteMultipartUpload => "s3:PutObject",
StorageAction::GetBucketPolicy => "s3:GetBucketPolicy",
StorageAction::PutBucketPolicy => "s3:PutBucketPolicy",
StorageAction::DeleteBucketPolicy => "s3:DeleteBucketPolicy",
}
}
}
@@ -54,7 +54,7 @@ impl S3Action {
/// Context extracted from a request, used for policy evaluation.
#[derive(Debug, Clone)]
pub struct RequestContext {
pub action: S3Action,
pub action: StorageAction,
pub bucket: Option<String>,
pub key: Option<String>,
}
@@ -70,7 +70,7 @@ impl RequestContext {
}
}
/// Resolve the S3 action from an incoming HTTP request.
/// Resolve the storage action from an incoming HTTP request.
pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
let method = req.method().clone();
let path = req.uri().path().to_string();
@@ -87,7 +87,7 @@ pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
0 => {
// Root: GET / -> ListBuckets
RequestContext {
action: S3Action::ListAllMyBuckets,
action: StorageAction::ListAllMyBuckets,
bucket: None,
key: None,
}
@@ -98,15 +98,15 @@ pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
let has_uploads = query.contains_key("uploads");
let action = match (&method, has_policy, has_uploads) {
(&Method::GET, true, _) => S3Action::GetBucketPolicy,
(&Method::PUT, true, _) => S3Action::PutBucketPolicy,
(&Method::DELETE, true, _) => S3Action::DeleteBucketPolicy,
(&Method::GET, _, true) => S3Action::ListBucketMultipartUploads,
(&Method::GET, _, _) => S3Action::ListBucket,
(&Method::PUT, _, _) => S3Action::CreateBucket,
(&Method::DELETE, _, _) => S3Action::DeleteBucket,
(&Method::HEAD, _, _) => S3Action::HeadBucket,
_ => S3Action::ListBucket,
(&Method::GET, true, _) => StorageAction::GetBucketPolicy,
(&Method::PUT, true, _) => StorageAction::PutBucketPolicy,
(&Method::DELETE, true, _) => StorageAction::DeleteBucketPolicy,
(&Method::GET, _, true) => StorageAction::ListBucketMultipartUploads,
(&Method::GET, _, _) => StorageAction::ListBucket,
(&Method::PUT, _, _) => StorageAction::CreateBucket,
(&Method::DELETE, _, _) => StorageAction::DeleteBucket,
(&Method::HEAD, _, _) => StorageAction::HeadBucket,
_ => StorageAction::ListBucket,
};
RequestContext {
@@ -125,16 +125,16 @@ pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
let has_uploads = query.contains_key("uploads");
let action = match &method {
&Method::PUT if has_part_number && has_upload_id => S3Action::UploadPart,
&Method::PUT if has_copy_source => S3Action::CopyObject,
&Method::PUT => S3Action::PutObject,
&Method::GET => S3Action::GetObject,
&Method::HEAD => S3Action::HeadObject,
&Method::DELETE if has_upload_id => S3Action::AbortMultipartUpload,
&Method::DELETE => S3Action::DeleteObject,
&Method::POST if has_uploads => S3Action::InitiateMultipartUpload,
&Method::POST if has_upload_id => S3Action::CompleteMultipartUpload,
_ => S3Action::GetObject,
&Method::PUT if has_part_number && has_upload_id => StorageAction::UploadPart,
&Method::PUT if has_copy_source => StorageAction::CopyObject,
&Method::PUT => StorageAction::PutObject,
&Method::GET => StorageAction::GetObject,
&Method::HEAD => StorageAction::HeadObject,
&Method::DELETE if has_upload_id => StorageAction::AbortMultipartUpload,
&Method::DELETE => StorageAction::DeleteObject,
&Method::POST if has_uploads => StorageAction::InitiateMultipartUpload,
&Method::POST if has_upload_id => StorageAction::CompleteMultipartUpload,
_ => StorageAction::GetObject,
};
RequestContext {
@@ -144,7 +144,7 @@ pub fn resolve_action(req: &Request<Incoming>) -> RequestContext {
}
}
_ => RequestContext {
action: S3Action::ListAllMyBuckets,
action: StorageAction::ListAllMyBuckets,
bucket: None,
key: None,
},

View File

@@ -4,8 +4,8 @@ use hyper::Request;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use crate::config::{Credential, S3Config};
use crate::s3_error::S3Error;
use crate::config::{Credential, SmartStorageConfig};
use crate::error::StorageError;
type HmacSha256 = Hmac<Sha256>;
@@ -27,8 +27,8 @@ struct SigV4Header {
/// Verify the request's SigV4 signature. Returns the caller identity on success.
pub fn verify_request(
req: &Request<Incoming>,
config: &S3Config,
) -> Result<AuthenticatedIdentity, S3Error> {
config: &SmartStorageConfig,
) -> Result<AuthenticatedIdentity, StorageError> {
let auth_header = req
.headers()
.get("authorization")
@@ -37,18 +37,18 @@ pub fn verify_request(
// Reject SigV2
if auth_header.starts_with("AWS ") {
return Err(S3Error::authorization_header_malformed());
return Err(StorageError::authorization_header_malformed());
}
if !auth_header.starts_with("AWS4-HMAC-SHA256") {
return Err(S3Error::authorization_header_malformed());
return Err(StorageError::authorization_header_malformed());
}
let parsed = parse_auth_header(auth_header)?;
// Look up credential
let credential = find_credential(&parsed.access_key_id, config)
.ok_or_else(S3Error::invalid_access_key_id)?;
.ok_or_else(StorageError::invalid_access_key_id)?;
// Get x-amz-date
let amz_date = req
@@ -60,7 +60,7 @@ pub fn verify_request(
.get("date")
.and_then(|v| v.to_str().ok())
})
.ok_or_else(|| S3Error::missing_security_header("Missing x-amz-date header"))?;
.ok_or_else(|| StorageError::missing_security_header("Missing x-amz-date header"))?;
// Enforce 15-min clock skew
check_clock_skew(amz_date)?;
@@ -99,7 +99,7 @@ pub fn verify_request(
// Constant-time comparison
if !constant_time_eq(computed_hex.as_bytes(), parsed.signature.as_bytes()) {
return Err(S3Error::signature_does_not_match());
return Err(StorageError::signature_does_not_match());
}
Ok(AuthenticatedIdentity {
@@ -108,11 +108,11 @@ pub fn verify_request(
}
/// Parse the Authorization header into its components.
fn parse_auth_header(header: &str) -> Result<SigV4Header, S3Error> {
fn parse_auth_header(header: &str) -> Result<SigV4Header, StorageError> {
// Format: AWS4-HMAC-SHA256 Credential=KEY/YYYYMMDD/region/s3/aws4_request, SignedHeaders=h1;h2, Signature=hex
let after_algo = header
.strip_prefix("AWS4-HMAC-SHA256")
.ok_or_else(S3Error::authorization_header_malformed)?
.ok_or_else(StorageError::authorization_header_malformed)?
.trim();
let mut credential_str = None;
@@ -131,17 +131,17 @@ fn parse_auth_header(header: &str) -> Result<SigV4Header, S3Error> {
}
let credential_str = credential_str
.ok_or_else(S3Error::authorization_header_malformed)?;
.ok_or_else(StorageError::authorization_header_malformed)?;
let signed_headers_str = signed_headers_str
.ok_or_else(S3Error::authorization_header_malformed)?;
.ok_or_else(StorageError::authorization_header_malformed)?;
let signature = signature_str
.ok_or_else(S3Error::authorization_header_malformed)?
.ok_or_else(StorageError::authorization_header_malformed)?
.to_string();
// Parse credential: KEY/YYYYMMDD/region/s3/aws4_request
let cred_parts: Vec<&str> = credential_str.splitn(5, '/').collect();
if cred_parts.len() < 5 {
return Err(S3Error::authorization_header_malformed());
return Err(StorageError::authorization_header_malformed());
}
let access_key_id = cred_parts[0].to_string();
@@ -163,7 +163,7 @@ fn parse_auth_header(header: &str) -> Result<SigV4Header, S3Error> {
}
/// Find a credential by access key ID.
fn find_credential<'a>(access_key_id: &str, config: &'a S3Config) -> Option<&'a Credential> {
fn find_credential<'a>(access_key_id: &str, config: &'a SmartStorageConfig) -> Option<&'a Credential> {
config
.auth
.credentials
@@ -172,17 +172,17 @@ fn find_credential<'a>(access_key_id: &str, config: &'a S3Config) -> Option<&'a
}
/// Check clock skew (15 minutes max).
fn check_clock_skew(amz_date: &str) -> Result<(), S3Error> {
fn check_clock_skew(amz_date: &str) -> Result<(), StorageError> {
// Parse ISO 8601 basic format: YYYYMMDDTHHMMSSZ
let parsed = chrono::NaiveDateTime::parse_from_str(amz_date, "%Y%m%dT%H%M%SZ")
.map_err(|_| S3Error::authorization_header_malformed())?;
.map_err(|_| StorageError::authorization_header_malformed())?;
let request_time = chrono::DateTime::<chrono::Utc>::from_naive_utc_and_offset(parsed, chrono::Utc);
let now = chrono::Utc::now();
let diff = (now - request_time).num_seconds().unsigned_abs();
if diff > 15 * 60 {
return Err(S3Error::request_time_too_skewed());
return Err(StorageError::request_time_too_skewed());
}
Ok(())

View File

@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct S3Config {
pub struct SmartStorageConfig {
pub server: ServerConfig,
pub storage: StorageConfig,
pub auth: AuthConfig,

View File

@@ -1,16 +1,14 @@
use hyper::{Response, StatusCode};
use http_body_util::Full;
use bytes::Bytes;
use hyper::StatusCode;
#[derive(Debug, thiserror::Error)]
#[error("S3Error({code}): {message}")]
pub struct S3Error {
#[error("StorageError({code}): {message}")]
pub struct StorageError {
pub code: String,
pub message: String,
pub status: StatusCode,
}
impl S3Error {
impl StorageError {
pub fn new(code: &str, message: &str, status: StatusCode) -> Self {
Self {
code: code.to_string(),
@@ -105,14 +103,4 @@ impl S3Error {
self.code, self.message
)
}
pub fn to_response(&self, request_id: &str) -> Response<Full<Bytes>> {
let xml = self.to_xml();
Response::builder()
.status(self.status)
.header("content-type", "application/xml")
.header("x-amz-request-id", request_id)
.body(Full::new(Bytes::from(xml)))
.unwrap()
}
}

View File

@@ -3,7 +3,7 @@ mod auth;
mod config;
mod management;
mod policy;
mod s3_error;
mod error;
mod server;
mod storage;
mod xml_response;
@@ -11,7 +11,7 @@ mod xml_response;
use clap::Parser;
#[derive(Parser)]
#[command(name = "rusts3", about = "High-performance S3-compatible server")]
#[command(name = "ruststorage", about = "High-performance S3-compatible storage server")]
struct Cli {
/// Run in management mode (IPC via stdin/stdout)
#[arg(long)]
@@ -38,7 +38,7 @@ async fn main() -> anyhow::Result<()> {
management::management_loop().await?;
} else {
eprintln!("rusts3: use --management flag for IPC mode");
eprintln!("ruststorage: use --management flag for IPC mode");
std::process::exit(1);
}

View File

@@ -4,8 +4,8 @@ use serde_json::Value;
use std::io::Write;
use tokio::io::{AsyncBufReadExt, BufReader};
use crate::config::S3Config;
use crate::server::S3Server;
use crate::config::SmartStorageConfig;
use crate::server::StorageServer;
#[derive(Deserialize)]
struct IpcRequest {
@@ -62,7 +62,7 @@ pub async fn management_loop() -> Result<()> {
data: serde_json::json!({}),
});
let mut server: Option<S3Server> = None;
let mut server: Option<StorageServer> = None;
let stdin = BufReader::new(tokio::io::stdin());
let mut lines = stdin.lines();
@@ -87,11 +87,11 @@ pub async fn management_loop() -> Result<()> {
"start" => {
#[derive(Deserialize)]
struct StartParams {
config: S3Config,
config: SmartStorageConfig,
}
match serde_json::from_value::<StartParams>(req.params) {
Ok(params) => {
match S3Server::start(params.config).await {
match StorageServer::start(params.config).await {
Ok(s) => {
server = Some(s);
send_response(id, serde_json::json!({}));

View File

@@ -6,7 +6,7 @@ use tokio::sync::RwLock;
use crate::action::RequestContext;
use crate::auth::AuthenticatedIdentity;
use crate::s3_error::S3Error;
use crate::error::StorageError;
// ============================
// Policy data model
@@ -284,50 +284,50 @@ fn simple_wildcard_match(pattern: &str, value: &str) -> bool {
const MAX_POLICY_SIZE: usize = 20 * 1024; // 20 KB
pub fn validate_policy(json: &str) -> Result<BucketPolicy, S3Error> {
pub fn validate_policy(json: &str) -> Result<BucketPolicy, StorageError> {
if json.len() > MAX_POLICY_SIZE {
return Err(S3Error::malformed_policy("Policy exceeds maximum size of 20KB"));
return Err(StorageError::malformed_policy("Policy exceeds maximum size of 20KB"));
}
let policy: BucketPolicy =
serde_json::from_str(json).map_err(|e| S3Error::malformed_policy(&e.to_string()))?;
serde_json::from_str(json).map_err(|e| StorageError::malformed_policy(&e.to_string()))?;
if policy.version != "2012-10-17" {
return Err(S3Error::malformed_policy(
return Err(StorageError::malformed_policy(
"Policy version must be \"2012-10-17\"",
));
}
if policy.statements.is_empty() {
return Err(S3Error::malformed_policy(
return Err(StorageError::malformed_policy(
"Policy must contain at least one statement",
));
}
for (i, stmt) in policy.statements.iter().enumerate() {
if stmt.action.is_empty() {
return Err(S3Error::malformed_policy(&format!(
return Err(StorageError::malformed_policy(&format!(
"Statement {} has no actions",
i
)));
}
for action in &stmt.action {
if action != "*" && !action.starts_with("s3:") {
return Err(S3Error::malformed_policy(&format!(
return Err(StorageError::malformed_policy(&format!(
"Action \"{}\" must start with \"s3:\"",
action
)));
}
}
if stmt.resource.is_empty() {
return Err(S3Error::malformed_policy(&format!(
return Err(StorageError::malformed_policy(&format!(
"Statement {} has no resources",
i
)));
}
for resource in &stmt.resource {
if resource != "*" && !resource.starts_with("arn:aws:s3:::") {
return Err(S3Error::malformed_policy(&format!(
return Err(StorageError::malformed_policy(&format!(
"Resource \"{}\" must start with \"arn:aws:s3:::\"",
resource
)));

View File

@@ -18,23 +18,22 @@ use tokio::sync::watch;
use tokio_util::io::ReaderStream;
use uuid::Uuid;
use crate::action::{self, RequestContext, S3Action};
use crate::action::{self, RequestContext, StorageAction};
use crate::auth::{self, AuthenticatedIdentity};
use crate::config::S3Config;
use crate::config::SmartStorageConfig;
use crate::policy::{self, PolicyDecision, PolicyStore};
use crate::s3_error::S3Error;
use crate::error::StorageError;
use crate::storage::FileStore;
use crate::xml_response;
pub struct S3Server {
pub struct StorageServer {
store: Arc<FileStore>,
config: S3Config,
shutdown_tx: watch::Sender<bool>,
server_handle: tokio::task::JoinHandle<()>,
}
impl S3Server {
pub async fn start(config: S3Config) -> Result<Self> {
impl StorageServer {
pub async fn start(config: SmartStorageConfig) -> Result<Self> {
let store = Arc::new(FileStore::new(config.storage.directory.clone().into()));
// Initialize or reset storage
@@ -105,12 +104,11 @@ impl S3Server {
});
if !config.server.silent {
tracing::info!("S3 server listening on {}", addr);
tracing::info!("Storage server listening on {}", addr);
}
Ok(Self {
store,
config,
shutdown_tx,
server_handle,
})
@@ -126,7 +124,7 @@ impl S3Server {
}
}
impl S3Config {
impl SmartStorageConfig {
fn address(&self) -> &str {
&self.server.address
}
@@ -194,7 +192,7 @@ fn empty_response(status: StatusCode, request_id: &str) -> Response<BoxBody> {
.unwrap()
}
fn s3_error_response(err: &S3Error, request_id: &str) -> Response<BoxBody> {
fn storage_error_response(err: &StorageError, request_id: &str) -> Response<BoxBody> {
let xml = err.to_xml();
Response::builder()
.status(err.status)
@@ -207,7 +205,7 @@ fn s3_error_response(err: &S3Error, request_id: &str) -> Response<BoxBody> {
async fn handle_request(
req: Request<Incoming>,
store: Arc<FileStore>,
config: S3Config,
config: SmartStorageConfig,
policy_store: Arc<PolicyStore>,
) -> Result<Response<BoxBody>, std::convert::Infallible> {
let request_id = Uuid::new_v4().to_string();
@@ -221,7 +219,7 @@ async fn handle_request(
return Ok(resp);
}
// Step 1: Resolve S3 action from request
// Step 1: Resolve storage action from request
let request_ctx = action::resolve_action(&req);
// Step 2: Auth + policy pipeline
@@ -240,7 +238,7 @@ async fn handle_request(
Ok(id) => Some(id),
Err(e) => {
tracing::warn!("Auth failed: {}", e.message);
return Ok(s3_error_response(&e, &request_id));
return Ok(storage_error_response(&e, &request_id));
}
}
} else {
@@ -250,7 +248,7 @@ async fn handle_request(
// Step 3: Authorization (policy evaluation)
if let Err(e) = authorize_request(&request_ctx, identity.as_ref(), &policy_store).await {
return Ok(s3_error_response(&e, &request_id));
return Ok(storage_error_response(&e, &request_id));
}
}
@@ -258,12 +256,12 @@ async fn handle_request(
let mut response = match route_request(req, store, &config, &request_id, &policy_store).await {
Ok(resp) => resp,
Err(err) => {
if let Some(s3err) = err.downcast_ref::<S3Error>() {
s3_error_response(s3err, &request_id)
if let Some(s3err) = err.downcast_ref::<StorageError>() {
storage_error_response(s3err, &request_id)
} else {
tracing::error!("Internal error: {}", err);
let s3err = S3Error::internal_error(&err.to_string());
s3_error_response(&s3err, &request_id)
let s3err = StorageError::internal_error(&err.to_string());
storage_error_response(&s3err, &request_id)
}
}
};
@@ -290,11 +288,11 @@ async fn authorize_request(
ctx: &RequestContext,
identity: Option<&AuthenticatedIdentity>,
policy_store: &PolicyStore,
) -> Result<(), S3Error> {
) -> Result<(), StorageError> {
// ListAllMyBuckets requires authentication (no bucket to apply policy to)
if ctx.action == S3Action::ListAllMyBuckets {
if ctx.action == StorageAction::ListAllMyBuckets {
if identity.is_none() {
return Err(S3Error::access_denied());
return Err(StorageError::access_denied());
}
return Ok(());
}
@@ -304,7 +302,7 @@ async fn authorize_request(
if let Some(bucket_policy) = policy_store.get_policy(bucket).await {
let decision = policy::evaluate_policy(&bucket_policy, ctx, identity);
match decision {
PolicyDecision::Deny => return Err(S3Error::access_denied()),
PolicyDecision::Deny => return Err(StorageError::access_denied()),
PolicyDecision::Allow => return Ok(()),
PolicyDecision::NoOpinion => {
// Fall through to default behavior
@@ -315,7 +313,7 @@ async fn authorize_request(
// Default: authenticated users get full access, anonymous denied
if identity.is_none() {
return Err(S3Error::access_denied());
return Err(StorageError::access_denied());
}
Ok(())
@@ -328,7 +326,7 @@ async fn authorize_request(
async fn route_request(
req: Request<Incoming>,
store: Arc<FileStore>,
_config: &S3Config,
_config: &SmartStorageConfig,
request_id: &str,
policy_store: &Arc<PolicyStore>,
) -> Result<Response<BoxBody>> {
@@ -416,8 +414,8 @@ async fn route_request(
let upload_id = query.get("uploadId").unwrap().clone();
handle_complete_multipart(req, store, &bucket, &key, &upload_id, request_id).await
} else {
let err = S3Error::invalid_request("Invalid POST request");
Ok(s3_error_response(&err, request_id))
let err = StorageError::invalid_request("Invalid POST request");
Ok(storage_error_response(&err, request_id))
}
}
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
@@ -469,7 +467,7 @@ async fn handle_head_bucket(
if store.bucket_exists(bucket).await {
Ok(empty_response(StatusCode::OK, request_id))
} else {
Err(S3Error::no_such_bucket().into())
Err(StorageError::no_such_bucket().into())
}
}
@@ -684,7 +682,7 @@ async fn handle_get_bucket_policy(
.unwrap();
Ok(resp)
}
None => Err(S3Error::no_such_bucket_policy().into()),
None => Err(StorageError::no_such_bucket_policy().into()),
}
}
@@ -697,7 +695,7 @@ async fn handle_put_bucket_policy(
) -> Result<Response<BoxBody>> {
// Verify bucket exists
if !store.bucket_exists(bucket).await {
return Err(S3Error::no_such_bucket().into());
return Err(StorageError::no_such_bucket().into());
}
// Read body
@@ -711,7 +709,7 @@ async fn handle_put_bucket_policy(
policy_store
.put_policy(bucket, validated_policy)
.await
.map_err(|e| S3Error::internal_error(&e.to_string()))?;
.map_err(|e| StorageError::internal_error(&e.to_string()))?;
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
}
@@ -724,7 +722,7 @@ async fn handle_delete_bucket_policy(
policy_store
.delete_policy(bucket)
.await
.map_err(|e| S3Error::internal_error(&e.to_string()))?;
.map_err(|e| StorageError::internal_error(&e.to_string()))?;
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
}
@@ -758,7 +756,7 @@ async fn handle_upload_part(
.unwrap_or(0);
if part_number < 1 || part_number > 10000 {
return Err(S3Error::invalid_part_number().into());
return Err(StorageError::invalid_part_number().into());
}
let body = req.into_body();
@@ -927,7 +925,7 @@ fn extract_xml_value<'a>(xml: &'a str, tag: &str) -> Option<String> {
// CORS
// ============================
fn build_cors_preflight(config: &S3Config, request_id: &str) -> Response<BoxBody> {
fn build_cors_preflight(config: &SmartStorageConfig, request_id: &str) -> Response<BoxBody> {
let mut builder = Response::builder()
.status(StatusCode::NO_CONTENT)
.header("x-amz-request-id", request_id);
@@ -951,7 +949,7 @@ fn build_cors_preflight(config: &S3Config, request_id: &str) -> Response<BoxBody
builder.body(empty_body()).unwrap()
}
fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &S3Config) {
fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &SmartStorageConfig) {
if let Some(ref origins) = config.cors.allowed_origins {
headers.insert(
"access-control-allow-origin",

View File

@@ -10,19 +10,17 @@ use tokio::fs;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufWriter};
use uuid::Uuid;
use crate::s3_error::S3Error;
use crate::error::StorageError;
// ============================
// Result types
// ============================
pub struct PutResult {
pub size: u64,
pub md5: String,
}
pub struct GetResult {
pub key: String,
pub size: u64,
pub last_modified: DateTime<Utc>,
pub md5: String,
@@ -32,7 +30,6 @@ pub struct GetResult {
}
pub struct HeadResult {
pub key: String,
pub size: u64,
pub last_modified: DateTime<Utc>,
pub md5: String,
@@ -40,7 +37,6 @@ pub struct HeadResult {
}
pub struct CopyResult {
pub size: u64,
pub md5: String,
pub last_modified: DateTime<Utc>,
}
@@ -69,14 +65,12 @@ pub struct BucketInfo {
pub struct MultipartUploadInfo {
pub upload_id: String,
pub bucket: String,
pub key: String,
pub initiated: DateTime<Utc>,
}
pub struct CompleteMultipartResult {
pub etag: String,
pub size: u64,
}
// ============================
@@ -126,10 +120,6 @@ impl FileStore {
self.root_dir.join(".policies")
}
pub fn policy_path(&self, bucket: &str) -> PathBuf {
self.policies_dir().join(format!("{}.policy.json", bucket))
}
pub async fn reset(&self) -> Result<()> {
if self.root_dir.exists() {
fs::remove_dir_all(&self.root_dir).await?;
@@ -184,13 +174,13 @@ impl FileStore {
let bucket_path = self.root_dir.join(bucket);
if !bucket_path.is_dir() {
return Err(S3Error::no_such_bucket().into());
return Err(StorageError::no_such_bucket().into());
}
// Check if bucket is empty (ignore hidden files)
let mut entries = fs::read_dir(&bucket_path).await?;
while let Some(_entry) = entries.next_entry().await? {
return Err(S3Error::bucket_not_empty().into());
return Err(StorageError::bucket_not_empty().into());
}
fs::remove_dir_all(&bucket_path).await?;
@@ -209,7 +199,7 @@ impl FileStore {
metadata: HashMap<String, String>,
) -> Result<PutResult> {
if !self.bucket_exists(bucket).await {
return Err(S3Error::no_such_bucket().into());
return Err(StorageError::no_such_bucket().into());
}
let object_path = self.object_path(bucket, key);
@@ -220,7 +210,6 @@ impl FileStore {
let file = fs::File::create(&object_path).await?;
let mut writer = BufWriter::new(file);
let mut hasher = Md5::new();
let mut total_size: u64 = 0;
// Stream body frames directly to file
let mut body = body;
@@ -229,7 +218,6 @@ impl FileStore {
Some(Ok(frame)) => {
if let Ok(data) = frame.into_data() {
hasher.update(&data);
total_size += data.len() as u64;
writer.write_all(&data).await?;
}
}
@@ -255,44 +243,6 @@ impl FileStore {
fs::write(&metadata_path, metadata_json).await?;
Ok(PutResult {
size: total_size,
md5: md5_hex,
})
}
pub async fn put_object_bytes(
&self,
bucket: &str,
key: &str,
data: &[u8],
metadata: HashMap<String, String>,
) -> Result<PutResult> {
if !self.bucket_exists(bucket).await {
return Err(S3Error::no_such_bucket().into());
}
let object_path = self.object_path(bucket, key);
if let Some(parent) = object_path.parent() {
fs::create_dir_all(parent).await?;
}
let mut hasher = Md5::new();
hasher.update(data);
let md5_hex = format!("{:x}", hasher.finalize());
fs::write(&object_path, data).await?;
// Write MD5 sidecar
let md5_path = format!("{}.md5", object_path.display());
fs::write(&md5_path, &md5_hex).await?;
// Write metadata sidecar
let metadata_path = format!("{}.metadata.json", object_path.display());
let metadata_json = serde_json::to_string_pretty(&metadata)?;
fs::write(&metadata_path, metadata_json).await?;
Ok(PutResult {
size: data.len() as u64,
md5: md5_hex,
})
}
@@ -306,7 +256,7 @@ impl FileStore {
let object_path = self.object_path(bucket, key);
if !object_path.exists() {
return Err(S3Error::no_such_key().into());
return Err(StorageError::no_such_key().into());
}
let file_meta = fs::metadata(&object_path).await?;
@@ -326,7 +276,6 @@ impl FileStore {
};
Ok(GetResult {
key: key.to_string(),
size,
last_modified,
md5,
@@ -340,7 +289,7 @@ impl FileStore {
let object_path = self.object_path(bucket, key);
if !object_path.exists() {
return Err(S3Error::no_such_key().into());
return Err(StorageError::no_such_key().into());
}
// Only stat the file, don't open it
@@ -352,7 +301,6 @@ impl FileStore {
let metadata = self.read_metadata(&object_path).await;
Ok(HeadResult {
key: key.to_string(),
size,
last_modified,
md5,
@@ -404,11 +352,11 @@ impl FileStore {
let dest_path = self.object_path(dest_bucket, dest_key);
if !src_path.exists() {
return Err(S3Error::no_such_key().into());
return Err(StorageError::no_such_key().into());
}
if !self.bucket_exists(dest_bucket).await {
return Err(S3Error::no_such_bucket().into());
return Err(StorageError::no_such_bucket().into());
}
if let Some(parent) = dest_path.parent() {
@@ -439,7 +387,6 @@ impl FileStore {
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
Ok(CopyResult {
size: file_meta.len(),
md5,
last_modified,
})
@@ -456,7 +403,7 @@ impl FileStore {
let bucket_path = self.root_dir.join(bucket);
if !bucket_path.is_dir() {
return Err(S3Error::no_such_bucket().into());
return Err(StorageError::no_such_bucket().into());
}
// Collect all object keys recursively
@@ -581,7 +528,7 @@ impl FileStore {
) -> Result<(String, u64)> {
let upload_dir = self.multipart_dir().join(upload_id);
if !upload_dir.is_dir() {
return Err(S3Error::no_such_upload().into());
return Err(StorageError::no_such_upload().into());
}
let part_path = upload_dir.join(format!("part-{}", part_number));
@@ -655,7 +602,7 @@ impl FileStore {
) -> Result<CompleteMultipartResult> {
let upload_dir = self.multipart_dir().join(upload_id);
if !upload_dir.is_dir() {
return Err(S3Error::no_such_upload().into());
return Err(StorageError::no_such_upload().into());
}
// Read metadata to get bucket/key
@@ -672,7 +619,6 @@ impl FileStore {
let dest_file = fs::File::create(&object_path).await?;
let mut writer = BufWriter::new(dest_file);
let mut hasher = Md5::new();
let mut total_size: u64 = 0;
for (part_number, _etag) in parts {
let part_path = upload_dir.join(format!("part-{}", part_number));
@@ -689,7 +635,6 @@ impl FileStore {
}
hasher.update(&buf[..n]);
writer.write_all(&buf[..n]).await?;
total_size += n as u64;
}
}
@@ -712,14 +657,13 @@ impl FileStore {
Ok(CompleteMultipartResult {
etag,
size: total_size,
})
}
pub async fn abort_multipart(&self, upload_id: &str) -> Result<()> {
let upload_dir = self.multipart_dir().join(upload_id);
if !upload_dir.is_dir() {
return Err(S3Error::no_such_upload().into());
return Err(StorageError::no_such_upload().into());
}
fs::remove_dir_all(&upload_dir).await?;
Ok(())
@@ -752,7 +696,6 @@ impl FileStore {
uploads.push(MultipartUploadInfo {
upload_id: meta.upload_id,
bucket: meta.bucket,
key: meta.key,
initiated,
});
@@ -772,7 +715,7 @@ impl FileStore {
let encoded = encode_key(key);
self.root_dir
.join(bucket)
.join(format!("{}._S3_object", encoded))
.join(format!("{}._storage_object", encoded))
}
async fn read_md5(&self, object_path: &Path) -> String {
@@ -832,7 +775,7 @@ impl FileStore {
if meta.is_dir() {
self.collect_keys(bucket_path, &entry.path(), keys).await?;
} else if name.ends_with("._S3_object")
} else if name.ends_with("._storage_object")
&& !name.ends_with(".metadata.json")
&& !name.ends_with(".md5")
{
@@ -842,7 +785,7 @@ impl FileStore {
.unwrap_or(Path::new(""))
.to_string_lossy()
.to_string();
let key = decode_key(relative.trim_end_matches("._S3_object"));
let key = decode_key(relative.trim_end_matches("._storage_object"));
keys.push(key);
}
}

View File

@@ -1,7 +1,7 @@
use crate::storage::{BucketInfo, ListObjectsResult, MultipartUploadInfo};
const XML_DECL: &str = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
const S3_NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
const STORAGE_NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
fn xml_escape(s: &str) -> String {
s.replace('&', "&amp;")
@@ -14,9 +14,9 @@ fn xml_escape(s: &str) -> String {
pub fn list_buckets_xml(buckets: &[BucketInfo]) -> String {
let mut xml = format!(
"{}\n<ListAllMyBucketsResult xmlns=\"{}\">\
<Owner><ID>123456789000</ID><DisplayName>S3rver</DisplayName></Owner>\
<Owner><ID>123456789000</ID><DisplayName>Storage</DisplayName></Owner>\
<Buckets>",
XML_DECL, S3_NS
XML_DECL, STORAGE_NS
);
for b in buckets {
@@ -39,7 +39,7 @@ pub fn list_objects_v1_xml(bucket: &str, result: &ListObjectsResult) -> String {
<MaxKeys>{}</MaxKeys>\
<IsTruncated>{}</IsTruncated>",
XML_DECL,
S3_NS,
STORAGE_NS,
xml_escape(bucket),
xml_escape(&result.prefix),
result.max_keys,
@@ -86,7 +86,7 @@ pub fn list_objects_v2_xml(bucket: &str, result: &ListObjectsResult) -> String {
<KeyCount>{}</KeyCount>\
<IsTruncated>{}</IsTruncated>",
XML_DECL,
S3_NS,
STORAGE_NS,
xml_escape(bucket),
xml_escape(&result.prefix),
result.max_keys,
@@ -132,15 +132,6 @@ pub fn list_objects_v2_xml(bucket: &str, result: &ListObjectsResult) -> String {
xml
}
pub fn error_xml(code: &str, message: &str) -> String {
format!(
"{}\n<Error><Code>{}</Code><Message>{}</Message></Error>",
XML_DECL,
xml_escape(code),
xml_escape(message)
)
}
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
format!(
"{}\n<CopyObjectResult>\
@@ -161,7 +152,7 @@ pub fn initiate_multipart_xml(bucket: &str, key: &str, upload_id: &str) -> Strin
<UploadId>{}</UploadId>\
</InitiateMultipartUploadResult>",
XML_DECL,
S3_NS,
STORAGE_NS,
xml_escape(bucket),
xml_escape(key),
xml_escape(upload_id)
@@ -177,7 +168,7 @@ pub fn complete_multipart_xml(bucket: &str, key: &str, etag: &str) -> String {
<ETag>\"{}\"</ETag>\
</CompleteMultipartUploadResult>",
XML_DECL,
S3_NS,
STORAGE_NS,
xml_escape(bucket),
xml_escape(key),
xml_escape(bucket),
@@ -195,7 +186,7 @@ pub fn list_multipart_uploads_xml(bucket: &str, uploads: &[MultipartUploadInfo])
<MaxUploads>1000</MaxUploads>\
<IsTruncated>false</IsTruncated>",
XML_DECL,
S3_NS,
STORAGE_NS,
xml_escape(bucket)
);
@@ -204,8 +195,8 @@ pub fn list_multipart_uploads_xml(bucket: &str, uploads: &[MultipartUploadInfo])
"<Upload>\
<Key>{}</Key>\
<UploadId>{}</UploadId>\
<Initiator><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Initiator>\
<Owner><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Owner>\
<Initiator><ID>STORAGE</ID><DisplayName>STORAGE</DisplayName></Initiator>\
<Owner><ID>STORAGE</ID><DisplayName>STORAGE</DisplayName></Owner>\
<StorageClass>STANDARD</StorageClass>\
<Initiated>{}</Initiated>\
</Upload>",

View File

@@ -12,9 +12,9 @@ import {
DeleteBucketPolicyCommand,
} from '@aws-sdk/client-s3';
import { Readable } from 'stream';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
let authClient: S3Client;
let wrongClient: S3Client;
@@ -35,8 +35,8 @@ async function streamToString(stream: Readable): Promise<string> {
// Server setup
// ============================
tap.test('should start S3 server with auth enabled', async () => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
tap.test('should start storage server with auth enabled', async () => {
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: {
port: TEST_PORT,
silent: true,
@@ -294,8 +294,8 @@ tap.test('authenticated: delete the bucket', async () => {
expect(response.$metadata.httpStatusCode).toEqual(204);
});
tap.test('should stop the S3 server', async () => {
await testSmarts3Instance.stop();
tap.test('should stop the storage server', async () => {
await testSmartStorageInstance.stop();
});
export default tap.start();

View File

@@ -1,9 +1,9 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
import { Readable } from 'stream';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
let s3Client: S3Client;
// Helper to convert stream to string
@@ -16,8 +16,8 @@ async function streamToString(stream: Readable): Promise<string> {
});
}
tap.test('should start the S3 server and configure client', async () => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
tap.test('should start the storage server and configure client', async () => {
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: {
port: 3337,
silent: true,
@@ -27,7 +27,7 @@ tap.test('should start the S3 server and configure client', async () => {
},
});
const descriptor = await testSmarts3Instance.getS3Descriptor();
const descriptor = await testSmartStorageInstance.getStorageDescriptor();
s3Client = new S3Client({
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
@@ -101,8 +101,8 @@ tap.test('should delete the bucket', async () => {
expect(response.$metadata.httpStatusCode).toEqual(204);
});
tap.test('should stop the S3 server', async () => {
await testSmarts3Instance.stop();
tap.test('should stop the storage server', async () => {
await testSmartStorageInstance.stop();
});
export default tap.start();

View File

@@ -14,9 +14,9 @@ import {
GetBucketPolicyCommand,
DeleteBucketPolicyCommand,
} from '@aws-sdk/client-s3';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
let authClient: S3Client;
const TEST_PORT = 3347;
@@ -56,7 +56,7 @@ function denyStatement(action: string) {
// ============================
tap.test('setup: start server, create bucket, upload object', async () => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
storage: { cleanSlate: true },
auth: {
@@ -275,7 +275,7 @@ tap.test('ListAllMyBuckets always requires auth → anonymous fetch to / returns
tap.test('Auth disabled mode → anonymous full access works', async () => {
// Start a second server with auth disabled
const noAuthInstance = await smarts3.Smarts3.createAndStart({
const noAuthInstance = await smartstorage.SmartStorage.createAndStart({
server: { port: 3348, silent: true, region: 'us-east-1' },
storage: { cleanSlate: true },
auth: { enabled: false, credentials: [] },
@@ -329,7 +329,7 @@ tap.test('teardown: clean up and stop server', async () => {
} catch {
// May already be deleted
}
await testSmarts3Instance.stop();
await testSmartStorageInstance.stop();
});
export default tap.start();

View File

@@ -7,9 +7,9 @@ import {
GetBucketPolicyCommand,
DeleteBucketPolicyCommand,
} from '@aws-sdk/client-s3';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
let authClient: S3Client;
const TEST_PORT = 3345;
@@ -33,8 +33,8 @@ const validStatement = {
// Server setup
// ============================
tap.test('setup: start S3 server with auth enabled', async () => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
tap.test('setup: start storage server with auth enabled', async () => {
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
storage: { cleanSlate: true },
auth: {
@@ -246,7 +246,7 @@ tap.test('Bucket deletion cleans up associated policy', async () => {
tap.test('teardown: delete bucket and stop server', async () => {
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
await testSmarts3Instance.stop();
await testSmartStorageInstance.stop();
});
export default tap.start();

View File

@@ -10,9 +10,9 @@ import {
DeleteBucketPolicyCommand,
} from '@aws-sdk/client-s3';
import { Readable } from 'stream';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
let authClient: S3Client;
const TEST_PORT = 3346;
@@ -48,7 +48,7 @@ async function clearPolicy() {
// ============================
tap.test('setup: start server, create bucket, upload object', async () => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: { port: TEST_PORT, silent: true, region: 'us-east-1' },
storage: { cleanSlate: true },
auth: {
@@ -511,7 +511,7 @@ tap.test('Policy allows s3:ListBucket → anonymous GET bucket (list objects) su
tap.test('teardown: clean up and stop server', async () => {
await authClient.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: 'test-obj.txt' }));
await authClient.send(new DeleteBucketCommand({ Bucket: BUCKET }));
await testSmarts3Instance.stop();
await testSmartStorageInstance.stop();
});
export default tap.start();

View File

@@ -1,12 +1,12 @@
import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as plugins from './plugins.js';
import * as smarts3 from '../ts/index.js';
import * as smartstorage from '../ts/index.js';
let testSmarts3Instance: smarts3.Smarts3;
let testSmartStorageInstance: smartstorage.SmartStorage;
tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
tap.test('should create a smartstorage instance and run it', async (toolsArg) => {
testSmartStorageInstance = await smartstorage.SmartStorage.createAndStart({
server: {
port: 3333,
},
@@ -20,7 +20,7 @@ tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
tap.test('should be able to access buckets', async () => {
const smartbucketInstance = new plugins.smartbucket.SmartBucket(
await testSmarts3Instance.getS3Descriptor(),
await testSmartStorageInstance.getStorageDescriptor(),
);
const bucket = await smartbucketInstance.createBucket('testbucket');
const baseDirectory = await bucket.getBaseDirectory();
@@ -31,7 +31,7 @@ tap.test('should be able to access buckets', async () => {
});
tap.test('should stop the instance', async () => {
await testSmarts3Instance.stop();
await testSmartStorageInstance.stop();
});
tap.start();

View File

@@ -2,7 +2,7 @@
* autocreated commitinfo by @push.rocks/commitinfo
*/
export const commitinfo = {
name: '@push.rocks/smarts3',
version: '5.2.0',
description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.'
name: '@push.rocks/smartstorage',
version: '6.0.0',
description: 'A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.'
}

View File

@@ -70,9 +70,9 @@ export interface IStorageConfig {
}
/**
* Complete smarts3 configuration
* Complete smartstorage configuration
*/
export interface ISmarts3Config {
export interface ISmartStorageConfig {
server?: IServerConfig;
storage?: IStorageConfig;
auth?: IAuthConfig;
@@ -85,7 +85,7 @@ export interface ISmarts3Config {
/**
* Default configuration values
*/
const DEFAULT_CONFIG: ISmarts3Config = {
const DEFAULT_CONFIG: ISmartStorageConfig = {
server: {
port: 3000,
address: '0.0.0.0',
@@ -100,8 +100,8 @@ const DEFAULT_CONFIG: ISmarts3Config = {
enabled: false,
credentials: [
{
accessKeyId: 'S3RVER',
secretAccessKey: 'S3RVER',
accessKeyId: 'STORAGE',
secretAccessKey: 'STORAGE',
},
],
},
@@ -133,7 +133,7 @@ const DEFAULT_CONFIG: ISmarts3Config = {
/**
* Merge user config with defaults (deep merge)
*/
function mergeConfig(userConfig: ISmarts3Config): Required<ISmarts3Config> {
function mergeConfig(userConfig: ISmartStorageConfig): Required<ISmartStorageConfig> {
return {
server: {
...DEFAULT_CONFIG.server!,
@@ -169,35 +169,35 @@ function mergeConfig(userConfig: ISmarts3Config): Required<ISmarts3Config> {
/**
* IPC command type map for RustBridge
*/
type TRustS3Commands = {
start: { params: { config: Required<ISmarts3Config> }; result: {} };
type TRustStorageCommands = {
start: { params: { config: Required<ISmartStorageConfig> }; result: {} };
stop: { params: {}; result: {} };
createBucket: { params: { name: string }; result: {} };
};
/**
* Main Smarts3 class - production-ready S3-compatible server
* Main SmartStorage class - production-ready S3-compatible storage server
*/
export class Smarts3 {
export class SmartStorage {
// STATIC
public static async createAndStart(configArg: ISmarts3Config = {}) {
const smartS3Instance = new Smarts3(configArg);
await smartS3Instance.start();
return smartS3Instance;
public static async createAndStart(configArg: ISmartStorageConfig = {}) {
const smartStorageInstance = new SmartStorage(configArg);
await smartStorageInstance.start();
return smartStorageInstance;
}
// INSTANCE
public config: Required<ISmarts3Config>;
private bridge: InstanceType<typeof plugins.RustBridge<TRustS3Commands>>;
public config: Required<ISmartStorageConfig>;
private bridge: InstanceType<typeof plugins.RustBridge<TRustStorageCommands>>;
constructor(configArg: ISmarts3Config = {}) {
constructor(configArg: ISmartStorageConfig = {}) {
this.config = mergeConfig(configArg);
this.bridge = new plugins.RustBridge<TRustS3Commands>({
binaryName: 'rusts3',
this.bridge = new plugins.RustBridge<TRustStorageCommands>({
binaryName: 'ruststorage',
localPaths: [
plugins.path.join(paths.packageDir, 'dist_rust', 'rusts3'),
plugins.path.join(paths.packageDir, 'rust', 'target', 'release', 'rusts3'),
plugins.path.join(paths.packageDir, 'rust', 'target', 'debug', 'rusts3'),
plugins.path.join(paths.packageDir, 'dist_rust', 'ruststorage'),
plugins.path.join(paths.packageDir, 'rust', 'target', 'release', 'ruststorage'),
plugins.path.join(paths.packageDir, 'rust', 'target', 'debug', 'ruststorage'),
],
readyTimeoutMs: 30000,
requestTimeoutMs: 300000,
@@ -207,21 +207,21 @@ export class Smarts3 {
public async start() {
const spawned = await this.bridge.spawn();
if (!spawned) {
throw new Error('Failed to spawn rusts3 binary. Make sure it is compiled (pnpm build).');
throw new Error('Failed to spawn ruststorage binary. Make sure it is compiled (pnpm build).');
}
await this.bridge.sendCommand('start', { config: this.config });
if (!this.config.server.silent) {
console.log('s3 server is running');
console.log('storage server is running');
}
}
public async getS3Descriptor(
public async getStorageDescriptor(
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
): Promise<plugins.tsclass.storage.IS3Descriptor> {
const cred = this.config.auth.credentials[0] || {
accessKeyId: 'S3RVER',
secretAccessKey: 'S3RVER',
accessKeyId: 'STORAGE',
secretAccessKey: 'STORAGE',
};
const descriptor: plugins.tsclass.storage.IS3Descriptor = {