Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0b9d8c4a72 | |||
| 65eb266983 | |||
| 54a0c2fb65 | |||
| 648ff98c2d | |||
| d6f178bde6 | |||
| ffaef5cb15 | |||
| d4cc1d43ea | |||
| 759becdd04 | |||
| 51e8836227 | |||
| 3c0a54e08b | |||
| c074a5d2ed | |||
| a9ba9de6be | |||
| 263e7a58b9 | |||
| 74b81d7ba8 | |||
| 0d4837184f | |||
| 7f3de92961 | |||
| a7bc902dd0 | |||
| 95d78d0d08 | |||
| b62cb0bc97 | |||
| 32346636e0 | |||
| 415ba3e76d | |||
| 6594f67d3e | |||
| 61974e0b54 | |||
| fc845956fa | |||
| eec1e09d2b | |||
| c3daf9d3f7 | |||
| 654f47b7fc | |||
| 18a2eb7e3f | |||
| 3ab667049a | |||
| 871f0e0b78 | |||
| f2bf3f1314 | |||
| a3b40923a5 | |||
| bdcfcee37a | |||
| d41878721c | |||
| 3f69d55f5f | |||
| ae015e433d | |||
| f86145f68b | |||
| 2b908c3e4b | |||
| dfb77842d3 | |||
| f13a99447d | |||
| aac5a11a7a | |||
| 36a1054b45 | |||
| 6aef7da9c4 | |||
| 1a1ceba76d | |||
| c931a0459a | |||
| a9eccfefc1 | |||
| 92a884354d | |||
| 008208dd0b | |||
| 94d60b8a73 | |||
| c41316eb28 | |||
| 8071a0d529 | |||
| a4c9679ccc | |||
| b16cfe0aac | |||
| 976f9ba8ca | |||
| 67c4f71a24 | |||
| dfadca9b80 | |||
| a7579d2e12 | |||
| de4632d186 | |||
| e69ca55a40 | |||
| 5803ef597f | |||
| 0bb6bfbb37 |
66
.gitea/workflows/default_nottags.yaml
Normal file
66
.gitea/workflows/default_nottags.yaml
Normal file
@@ -0,0 +1,66 @@
|
||||
name: Default (not tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags-ignore:
|
||||
- '**'
|
||||
|
||||
env:
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Install pnpm and npmci
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @ship.zone/npmci
|
||||
|
||||
- name: Run npm prepare
|
||||
run: npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
124
.gitea/workflows/default_tags.yaml
Normal file
124
.gitea/workflows/default_tags.yaml
Normal file
@@ -0,0 +1,124 @@
|
||||
name: Default (tags)
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
env:
|
||||
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
|
||||
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@/${{gitea.repository}}.git
|
||||
NPMCI_TOKEN_NPM: ${{secrets.NPMCI_TOKEN_NPM}}
|
||||
NPMCI_TOKEN_NPM2: ${{secrets.NPMCI_TOKEN_NPM2}}
|
||||
NPMCI_GIT_GITHUBTOKEN: ${{secrets.NPMCI_GIT_GITHUBTOKEN}}
|
||||
NPMCI_URL_CLOUDLY: ${{secrets.NPMCI_URL_CLOUDLY}}
|
||||
|
||||
jobs:
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Audit production dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --prod
|
||||
continue-on-error: true
|
||||
|
||||
- name: Audit development dependencies
|
||||
run: |
|
||||
npmci command npm config set registry https://registry.npmjs.org
|
||||
npmci command pnpm audit --audit-level=high --dev
|
||||
continue-on-error: true
|
||||
|
||||
test:
|
||||
if: ${{ always() }}
|
||||
needs: security
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Test stable
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm test
|
||||
|
||||
- name: Test build
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
npmci npm build
|
||||
|
||||
release:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Release
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm publish
|
||||
|
||||
metadata:
|
||||
needs: test
|
||||
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ${{ env.IMAGE }}
|
||||
continue-on-error: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Prepare
|
||||
run: |
|
||||
pnpm install -g pnpm
|
||||
pnpm install -g @ship.zone/npmci
|
||||
npmci npm prepare
|
||||
|
||||
- name: Code quality
|
||||
run: |
|
||||
npmci command npm install -g typescript
|
||||
npmci npm install
|
||||
|
||||
- name: Trigger
|
||||
run: npmci trigger
|
||||
|
||||
- name: Build docs and upload artifacts
|
||||
run: |
|
||||
npmci node install stable
|
||||
npmci npm install
|
||||
pnpm install -g @git.zone/tsdoc
|
||||
npmci command tsdoc
|
||||
continue-on-error: true
|
||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -3,7 +3,6 @@
|
||||
# artifacts
|
||||
coverage/
|
||||
public/
|
||||
pages/
|
||||
|
||||
# installs
|
||||
node_modules/
|
||||
@@ -17,4 +16,9 @@ node_modules/
|
||||
dist/
|
||||
dist_*/
|
||||
|
||||
# custom
|
||||
# AI
|
||||
.claude/
|
||||
.serena/
|
||||
|
||||
#------# custom
|
||||
rust/target
|
||||
141
.gitlab-ci.yml
141
.gitlab-ci.yml
@@ -1,141 +0,0 @@
|
||||
# gitzone ci_default
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
|
||||
cache:
|
||||
paths:
|
||||
- .npmci_cache/
|
||||
key: '$CI_BUILD_STAGE'
|
||||
|
||||
stages:
|
||||
- security
|
||||
- test
|
||||
- release
|
||||
- metadata
|
||||
|
||||
before_script:
|
||||
- npm install -g @shipzone/npmci
|
||||
|
||||
# ====================
|
||||
# security stage
|
||||
# ====================
|
||||
mirror:
|
||||
stage: security
|
||||
script:
|
||||
- npmci git mirror
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
auditProductionDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --production --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=prod --production
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
auditDevDependencies:
|
||||
image: registry.gitlab.com/hosttoday/ht-docker-node:npmci
|
||||
stage: security
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci command npm install --ignore-scripts
|
||||
- npmci command npm config set registry https://registry.npmjs.org
|
||||
- npmci command npm audit --audit-level=high --only=dev
|
||||
tags:
|
||||
- docker
|
||||
allow_failure: true
|
||||
|
||||
# ====================
|
||||
# test stage
|
||||
# ====================
|
||||
|
||||
testStable:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci npm test
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
testBuild:
|
||||
stage: test
|
||||
script:
|
||||
- npmci npm prepare
|
||||
- npmci node install stable
|
||||
- npmci npm install
|
||||
- npmci command npm run build
|
||||
coverage: /\d+.?\d+?\%\s*coverage/
|
||||
tags:
|
||||
- docker
|
||||
|
||||
release:
|
||||
stage: release
|
||||
script:
|
||||
- npmci node install stable
|
||||
- npmci npm publish
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
# ====================
|
||||
# metadata stage
|
||||
# ====================
|
||||
codequality:
|
||||
stage: metadata
|
||||
allow_failure: true
|
||||
only:
|
||||
- tags
|
||||
script:
|
||||
- npmci command npm install -g tslint typescript
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command "tslint -c tslint.json ./ts/**/*.ts"
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- priv
|
||||
|
||||
trigger:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci trigger
|
||||
only:
|
||||
- tags
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
|
||||
pages:
|
||||
stage: metadata
|
||||
script:
|
||||
- npmci node install lts
|
||||
- npmci command npm install -g @gitzone/tsdoc
|
||||
- npmci npm prepare
|
||||
- npmci npm install
|
||||
- npmci command tsdoc
|
||||
tags:
|
||||
- lossless
|
||||
- docker
|
||||
- notpriv
|
||||
only:
|
||||
- tags
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- public
|
||||
allow_failure: true
|
||||
220
changelog.md
Normal file
220
changelog.md
Normal file
@@ -0,0 +1,220 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-02-13 - 5.1.1 - fix(smarts3)
|
||||
replace TypeScript server with Rust-powered core and IPC bridge
|
||||
|
||||
- Major refactor: Node.js/TypeScript in-process server replaced by a Rust crate ('rusts3') with a TypeScript IPC wrapper (RustBridge).
|
||||
- Removed many TypeScript server modules (smarts3-server, filesystem-store, multipart-manager, controllers, router, context, logger, xml utils, etc.); Smarts3Server export removed — public API now proxies to the Rust binary.
|
||||
- Smarts3 now spawns and communicates with the rusts3 binary via RustBridge IPC (commands include start, stop, createBucket).
|
||||
- Build & packaging changes: build script now runs `tsrust` before `tsbuild`; added `@git.zone/tsrust` devDependency; added `dist_rust` artifacts and new cross-compile targets in npmextra.json; .gitignore updated for rust/target.
|
||||
- Dependency changes: added `@push.rocks/smartrust` (RustBridge) and simplified plugins surface; previous smartfs/smartxml usage removed from TS code and replaced by the Rust implementation + IPC.
|
||||
- Added Rust project files (rust/Cargo.toml, rust/src/*) implementing server, IPC management loop, storage, XML responses, errors, and config.
|
||||
- Documentation updated (README and hints) to describe the Rust core, supported prebuilt targets (linux_amd64, linux_arm64), IPC commands, and developer build notes.
|
||||
|
||||
## 2025-11-23 - 5.1.0 - feat(multipart)
|
||||
Implement full multipart upload support with persistent manager, periodic cleanup, and API integration
|
||||
|
||||
- Add IMultipartConfig to server config with defaults (expirationDays: 7, cleanupIntervalMinutes: 60) and merge into existing config flow
|
||||
- Introduce MultipartUploadManager: persistent upload metadata on disk, part upload/assembly, restore uploads on startup, listParts/listUploads, abort/cleanup functionality
|
||||
- Start and stop multipart cleanup task from Smarts3Server lifecycle (startCleanupTask on start, stopCleanupTask on stop) with configurable interval and expiration
|
||||
- ObjectController: support multipart endpoints (initiate, upload part, complete, abort) and move assembled final object into the object store on completion; set ETag headers and return proper XML responses
|
||||
- BucketController: support listing in-progress multipart uploads via ?uploads query parameter and return S3-compatible XML
|
||||
- Persist multipart state to disk and restore on initialization to survive restarts; perform automatic cleanup of expired uploads
|
||||
|
||||
## 2025-11-23 - 5.0.2 - fix(readme)
|
||||
Clarify contribution agreement requirement in README
|
||||
|
||||
- Updated the Issue Reporting and Security section in readme.md to make it explicit that developers must sign and comply with the contribution agreement (and complete identification) before obtaining a code.foss.global account to submit pull requests.
|
||||
|
||||
## 2025-11-23 - 5.0.1 - fix(docs)
|
||||
Clarify README wording about S3 compatibility and AWS SDK usage
|
||||
|
||||
- Update README wording to "Full S3 API compatibility" and clarify it works seamlessly with AWS SDK v3 and other S3 clients
|
||||
|
||||
## 2025-11-23 - 5.0.0 - BREAKING CHANGE(core)
|
||||
Production-ready S3-compatible server: nested config, multipart uploads, CORS, structured logging, SmartFS migration and improved error handling
|
||||
|
||||
- Breaking change: configuration format migrated from flat to nested structure (server, storage, auth, cors, logging, limits). Update existing configs accordingly.
|
||||
- Implemented full multipart upload support (initiate, upload part, complete, abort) with on-disk part management and final assembly.
|
||||
- Added CORS middleware with configurable origins, methods, headers, exposed headers, maxAge and credentials support.
|
||||
- Structured, configurable logging (levels: error|warn|info|debug; formats: text|json) and request/response logging middleware.
|
||||
- Simple static credential authentication middleware (configurable list of credentials).
|
||||
- Migrated filesystem operations to @push.rocks/smartfs (Web Streams interoperability) and removed smartbucket from production dependencies.
|
||||
- Improved S3-compatible error handling and XML responses (S3Error class and XML utilities).
|
||||
- Exposed Smarts3Server and made store/multipart managers accessible for tests and advanced usage; added helper methods like getS3Descriptor and createBucket.
|
||||
|
||||
## 2025-11-23 - 4.0.0 - BREAKING CHANGE(Smarts3)
|
||||
Migrate Smarts3 configuration to nested server/storage objects and remove legacy flat config support
|
||||
|
||||
- Smarts3.createAndStart() and Smarts3 constructor now accept ISmarts3Config with nested `server` and `storage` objects.
|
||||
- Removed support for the legacy flat config shape (top-level `port` and `cleanSlate`) / ILegacySmarts3Config.
|
||||
- Updated tests to use new config shape (server:{ port, silent } and storage:{ cleanSlate }).
|
||||
- mergeConfig and Smarts3Server now rely on the nested config shape; consumers must update their initialization code.
|
||||
|
||||
## 2025-11-23 - 3.2.0 - feat(multipart)
|
||||
Add multipart upload support with MultipartUploadManager and controller integration
|
||||
|
||||
- Introduce MultipartUploadManager (ts/classes/multipart-manager.ts) to manage multipart upload lifecycle and store parts on disk
|
||||
- Wire multipart manager into server and request context (S3Context, Smarts3Server) and initialize multipart storage on server start
|
||||
- Add multipart-related routes and handlers in ObjectController: initiate (POST ?uploads), upload part (PUT ?partNumber&uploadId), complete (POST ?uploadId), and abort (DELETE ?uploadId)
|
||||
- On complete, combine parts into final object and store via existing FilesystemStore workflow
|
||||
- Expose multipart manager on Smarts3Server for controller access
|
||||
|
||||
## 2025-11-23 - 3.1.0 - feat(logging)
|
||||
Add structured Logger and integrate into Smarts3Server; pass full config to server
|
||||
|
||||
- Introduce a new Logger class (ts/classes/logger.ts) providing leveled logging (error, warn, info, debug), text/json formats and an enable flag.
|
||||
- Integrate Logger into Smarts3Server: use structured logging for server lifecycle events, HTTP request/response logging and S3 errors instead of direct console usage.
|
||||
- Smarts3 now passes the full merged configuration into Smarts3Server (config.logging can control logging behavior).
|
||||
- Server start/stop messages and internal request/error logs are emitted via the Logger and respect the configured logging level/format and silent option.
|
||||
|
||||
## 2025-11-23 - 3.0.4 - fix(smarts3)
|
||||
Use filesystem store for bucket creation and remove smartbucket runtime dependency
|
||||
|
||||
- Switched createBucket to call the internal FilesystemStore.createBucket instead of using @push.rocks/smartbucket
|
||||
- Made Smarts3Server.store public so Smarts3 can access the filesystem store directly
|
||||
- Removed runtime import/export of @push.rocks/smartbucket from plugins and moved @push.rocks/smartbucket to devDependencies in package.json
|
||||
- Updated createBucket to return a simple { name } object after creating the bucket via the filesystem store
|
||||
|
||||
## 2025-11-23 - 3.0.3 - fix(filesystem)
|
||||
Migrate filesystem implementation to @push.rocks/smartfs and add Web Streams handling
|
||||
|
||||
- Replace dependency @push.rocks/smartfile with @push.rocks/smartfs and update README references
|
||||
- plugins: instantiate SmartFs with SmartFsProviderNode and export smartfs (remove direct fs export)
|
||||
- Refactor FilesystemStore to use smartfs directory/file APIs for initialize, reset, list, read, write, copy and delete
|
||||
- Implement Web Stream ↔ Node.js stream conversion for uploads/downloads (Readable.fromWeb and writer.write with Uint8Array)
|
||||
- Persist and read metadata (.metadata.json) and cached MD5 (.md5) via smartfs APIs
|
||||
- Update readme.hints and documentation to note successful migration and next steps
|
||||
|
||||
## 2025-11-21 - 3.0.2 - fix(smarts3)
|
||||
Prepare patch release 3.0.2 — no code changes detected
|
||||
|
||||
- No source changes in the diff
|
||||
- Bump patch version from 3.0.1 to 3.0.2 for maintenance/release bookkeeping
|
||||
|
||||
## 2025-11-21 - 3.0.1 - fix(readme)
|
||||
Add Issue Reporting and Security section to README
|
||||
|
||||
- Add guidance to report bugs, issues, and security vulnerabilities via community.foss.global
|
||||
- Inform developers how to sign a contribution agreement and get a code.foss.global account to submit pull requests
|
||||
|
||||
## 2025-11-21 - 3.0.0 - BREAKING CHANGE(Smarts3)
|
||||
Remove legacy s3rver backend, simplify Smarts3 server API, and bump dependencies
|
||||
|
||||
- Remove legacy s3rver backend: s3rver and its types were removed from dependencies and are no longer exported from plugins.
|
||||
- Simplify Smarts3 API: removed useCustomServer option; Smarts3 now always uses the built-in Smarts3Server (s3Instance is Smarts3Server) and stop() always calls Smarts3Server.stop().
|
||||
- Update README to remove legacy s3rver compatibility mention.
|
||||
- Dependency updates: bumped @push.rocks/smartbucket to ^4.3.0 and @push.rocks/smartxml to ^2.0.0 (major upgrades), removed s3rver/@types/s3rver, bumped @aws-sdk/client-s3 to ^3.937.0 and @git.zone/tstest to ^3.1.0.
|
||||
|
||||
## 2025-11-21 - 2.3.0 - feat(smarts3-server)
|
||||
Introduce native custom S3 server implementation (Smarts3Server) with routing, middleware, context, filesystem store, controllers and XML utilities; add SmartXml and AWS SDK test; keep optional legacy s3rver backend.
|
||||
|
||||
- Add Smarts3Server: native, Node.js http-based S3-compatible server (ts/classes/smarts3-server.ts)
|
||||
- New routing and middleware system: S3Router and MiddlewareStack for pattern matching and middleware composition (ts/classes/router.ts, ts/classes/middleware-stack.ts)
|
||||
- Introduce request context and helpers: S3Context for parsing requests, sending responses and XML (ts/classes/context.ts)
|
||||
- Filesystem-backed storage: FilesystemStore with bucket/object operations, streaming uploads, MD5 handling and Windows-safe key encoding (ts/classes/filesystem-store.ts)
|
||||
- S3 error handling: S3Error class that maps S3 error codes and produces XML error responses (ts/classes/s3-error.ts)
|
||||
- Controllers for service, bucket and object operations with S3-compatible XML responses and copy/range support (ts/controllers/*.ts)
|
||||
- XML utilities and SmartXml integration for consistent XML generation/parsing (ts/utils/xml.utils.ts, ts/plugins.ts)
|
||||
- Expose native plugins (http, crypto, url, fs) and SmartXml via plugins.ts
|
||||
- ts/index.ts: add useCustomServer option, default to custom server, export Smarts3Server and handle start/stop for both custom and legacy backends
|
||||
- Add AWS SDK v3 integration test (test/test.aws-sdk.node.ts) to validate compatibility
|
||||
- package.json: add @aws-sdk/client-s3 devDependency and @push.rocks/smartxml dependency
|
||||
- Documentation: readme.md updated to describe native custom server and legacy s3rver compatibility
|
||||
|
||||
## 2025-11-20 - 2.2.7 - fix(core)
|
||||
Update dependencies, code style and project config; add pnpm overrides and ignore AI folders
|
||||
|
||||
- Bump devDependencies and runtime dependencies (@git.zone/*, @push.rocks/*, @tsclass/tsclass, s3rver) to newer compatible versions
|
||||
- Add pnpm.overrides entry to package.json and normalize repository URL format
|
||||
- Code style and formatting fixes in TypeScript sources (ts/index.ts, ts/00_commitinfo_data.ts): whitespace, trailing commas, parameter formatting and minor API-return typing preserved
|
||||
- tsconfig.json: simplify compiler options and compact exclude list
|
||||
- Update .gitignore to add AI-related folders (.claude/, .serena/) to avoid accidental commits
|
||||
- Documentation and changelog formatting tweaks (readme.md, changelog.md, npmextra.json) — whitespace/newline cleanups and expanded changelog entries
|
||||
|
||||
## 2025-08-16 - 2.2.6 - fix(Smarts3)
|
||||
|
||||
Allow overriding S3 descriptor; update dependencies, test config and documentation
|
||||
|
||||
- ts/index.ts: getS3Descriptor now accepts an optional Partial<IS3Descriptor> to override defaults (backwards compatible)
|
||||
- package.json: updated devDependencies and runtime dependency versions (tstest, smartpath, tsclass, s3rver, etc.) and added packageManager field
|
||||
- package.json: expanded test script to run tstest with --web --verbose --logfile --timeout 60
|
||||
- test/test.ts: test instance port changed to 3333
|
||||
- readme.md: major rewrite and expansion of usage examples, API reference and guides
|
||||
- added project config files: .claude/settings.local.json and .serena/project.yml
|
||||
|
||||
## 2024-11-06 - 2.2.5 - fix(ci)
|
||||
|
||||
Corrected docker image URLs in Gitea workflows to match the correct domain format.
|
||||
|
||||
- Updated IMAGE environment variable in .gitea/workflows/default_nottags.yaml
|
||||
- Updated IMAGE environment variable in .gitea/workflows/default_tags.yaml
|
||||
|
||||
## 2024-11-06 - 2.2.4 - fix(core)
|
||||
|
||||
Improve code style and update dependencies
|
||||
|
||||
- Updated @push.rocks/tapbundle to version ^5.4.3 in package.json.
|
||||
- Fixed markdown formatting in readme.md.
|
||||
- Improved code consistency in ts/00_commitinfo_data.ts, ts/plugins.ts, and test/test.ts.
|
||||
|
||||
## 2024-11-06 - 2.2.3 - fix(core)
|
||||
|
||||
Fix endpoint address from 'localhost' to '127.0.0.1' for better compatibility in Smarts3.getS3Descriptor
|
||||
|
||||
- Corrected the endpoint address in Smarts3.getS3Descriptor to ensure proper functioning across different environments.
|
||||
|
||||
## 2024-11-06 - 2.2.2 - fix(core)
|
||||
|
||||
Fixed function call for fastPut in the test suite to ensure proper file upload handling.
|
||||
|
||||
- Updated dependencies in package.json to newer versions.
|
||||
- Corrected the function call in test suite for file upload.
|
||||
|
||||
## 2024-10-26 - 2.2.1 - fix(core)
|
||||
|
||||
Fix import and typings for improved compatibility
|
||||
|
||||
- Corrected the type signature for `getS3Descriptor` to return `IS3Descriptor`.
|
||||
- Fixed import structure and updated dependencies for consistent namespace usage across plugins.
|
||||
|
||||
## 2024-10-26 - 2.2.0 - feat(ci)
|
||||
|
||||
Migrate CI/CD workflow from GitLab CI to Gitea CI
|
||||
|
||||
- Added new Gitea CI workflows for both non-tag and tag-based pushes
|
||||
- Removed existing GitLab CI configuration
|
||||
|
||||
## 2024-05-29 - 2.1.1 - Updates and minor changes
|
||||
|
||||
Updates and changes based on minor configuration improvements and organizational shifts.
|
||||
|
||||
- Updated description file.
|
||||
- Updated tsconfig settings.
|
||||
- Updated npmextra.json with new githost configuration.
|
||||
- Shifted to new organizational scheme.
|
||||
|
||||
## 2022-07-30 - 2.1.0 - Core improvements and fixes
|
||||
|
||||
Minor improvements and important core changes.
|
||||
|
||||
- Removed tslint from the core setup.
|
||||
|
||||
## 2022-07-30 - 2.0.2 - Bucket creation improvement
|
||||
|
||||
Enhanced file structure management.
|
||||
|
||||
- Improved bucket creation to store locally within the .nogit directory.
|
||||
|
||||
## 2022-04-14 - 2.0.0 to 2.0.1 - Structural updates and fixes
|
||||
|
||||
This release focused on core updates and structural changes.
|
||||
|
||||
- Reformatted the project structure.
|
||||
- Core updates with minor fixes.
|
||||
|
||||
## 2021-12-20 - 1.0.10 - ESM Transition
|
||||
|
||||
Breaking changes and minor fixes, transitioning to ES Modules.
|
||||
|
||||
- BREAKING CHANGE: Transitioned core setup to ESM.
|
||||
@@ -1,18 +1,51 @@
|
||||
{
|
||||
"gitzone": {
|
||||
"@git.zone/tsrust": {
|
||||
"targets": [
|
||||
"linux_amd64",
|
||||
"linux_arm64"
|
||||
]
|
||||
},
|
||||
"@git.zone/cli": {
|
||||
"projectType": "npm",
|
||||
"module": {
|
||||
"githost": "gitlab.com",
|
||||
"gitscope": "pushrocks",
|
||||
"githost": "code.foss.global",
|
||||
"gitscope": "push.rocks",
|
||||
"gitrepo": "smarts3",
|
||||
"shortDescription": "create an s3 endpoint that maps to a local directory",
|
||||
"npmPackagename": "@pushrocks/smarts3",
|
||||
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
|
||||
"npmPackagename": "@push.rocks/smarts3",
|
||||
"license": "MIT",
|
||||
"projectDomain": "push.rocks"
|
||||
"projectDomain": "push.rocks",
|
||||
"keywords": [
|
||||
"S3 Mock Server",
|
||||
"Local S3",
|
||||
"Node.js",
|
||||
"TypeScript",
|
||||
"Local Development",
|
||||
"Testing",
|
||||
"Cloud Storage",
|
||||
"File Storage",
|
||||
"AWS S3 Compatibility",
|
||||
"Development Tool",
|
||||
"S3 Endpoint",
|
||||
"S3 Simulation",
|
||||
"Bucket Management",
|
||||
"File Upload",
|
||||
"CI/CD Integration",
|
||||
"Developer Onboarding"
|
||||
]
|
||||
},
|
||||
"release": {
|
||||
"registries": [
|
||||
"https://verdaccio.lossless.digital",
|
||||
"https://registry.npmjs.org"
|
||||
],
|
||||
"accessLevel": "public"
|
||||
}
|
||||
},
|
||||
"npmci": {
|
||||
"npmGlobalTools": [],
|
||||
"npmAccessLevel": "public"
|
||||
"@git.zone/tsdoc": {
|
||||
"legal": "\n## License and Legal Information\n\nThis repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. \n\n**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.\n\n### Trademarks\n\nThis project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH and are not included within the scope of the MIT license granted herein. Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines, and any usage must be approved in writing by Task Venture Capital GmbH.\n\n### Company Information\n\nTask Venture Capital GmbH \nRegistered at District court Bremen HRB 35230 HB, Germany\n\nFor any legal inquiries or if you require further information, please contact us via email at hello@task.vc.\n\nBy using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.\n"
|
||||
},
|
||||
"@ship.zone/szci": {
|
||||
"npmGlobalTools": []
|
||||
}
|
||||
}
|
||||
29006
package-lock.json
generated
29006
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
65
package.json
65
package.json
@@ -1,24 +1,27 @@
|
||||
{
|
||||
"name": "@pushrocks/smarts3",
|
||||
"version": "1.0.10",
|
||||
"name": "@push.rocks/smarts3",
|
||||
"version": "5.1.1",
|
||||
"private": false,
|
||||
"description": "create an s3 endpoint that maps to a local directory",
|
||||
"description": "A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.",
|
||||
"main": "dist_ts/index.js",
|
||||
"typings": "dist_ts/index.d.ts",
|
||||
"type": "module",
|
||||
"author": "Lossless GmbH",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"test": "(tstest test/ --web)",
|
||||
"build": "(tsbuild --web)"
|
||||
"test": "(tstest test/ --web --verbose --logfile --timeout 60)",
|
||||
"build": "(tsrust && tsbuild --web --allowimplicitany)",
|
||||
"buildDocs": "tsdoc"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@gitzone/tsbuild": "^2.1.25",
|
||||
"@gitzone/tsbundle": "^1.0.78",
|
||||
"@gitzone/tstest": "^1.0.44",
|
||||
"@pushrocks/tapbundle": "^3.2.9",
|
||||
"@types/node": "^17.0.0",
|
||||
"tslint": "^6.1.3",
|
||||
"tslint-config-prettier": "^1.15.0"
|
||||
"@aws-sdk/client-s3": "^3.937.0",
|
||||
"@git.zone/tsbuild": "^3.1.0",
|
||||
"@git.zone/tsbundle": "^2.5.2",
|
||||
"@git.zone/tsrun": "^2.0.0",
|
||||
"@git.zone/tstest": "^3.1.0",
|
||||
"@push.rocks/smartbucket": "^4.3.0",
|
||||
"@git.zone/tsrust": "^1.3.0",
|
||||
"@types/node": "^22.9.0"
|
||||
},
|
||||
"browserslist": [
|
||||
"last 1 chrome versions"
|
||||
@@ -29,6 +32,7 @@
|
||||
"dist/**/*",
|
||||
"dist_*/**/*",
|
||||
"dist_ts/**/*",
|
||||
"dist_rust/**/*",
|
||||
"dist_ts_web/**/*",
|
||||
"assets/**/*",
|
||||
"cli.js",
|
||||
@@ -36,9 +40,38 @@
|
||||
"readme.md"
|
||||
],
|
||||
"dependencies": {
|
||||
"@pushrocks/smartbucket": "^1.0.44",
|
||||
"@pushrocks/smartfile": "^9.0.3",
|
||||
"@types/s3rver": "^3.7.0",
|
||||
"s3rver": "^3.7.1"
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartrust": "^1.0.0",
|
||||
"@tsclass/tsclass": "^9.3.0"
|
||||
},
|
||||
"keywords": [
|
||||
"S3 Mock Server",
|
||||
"Local S3",
|
||||
"Node.js",
|
||||
"TypeScript",
|
||||
"Local Development",
|
||||
"Testing",
|
||||
"Cloud Storage",
|
||||
"File Storage",
|
||||
"AWS S3 Compatibility",
|
||||
"Development Tool",
|
||||
"S3 Endpoint",
|
||||
"S3 Simulation",
|
||||
"Bucket Management",
|
||||
"File Upload",
|
||||
"CI/CD Integration",
|
||||
"Developer Onboarding"
|
||||
],
|
||||
"homepage": "https://code.foss.global/push.rocks/smarts3#readme",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://code.foss.global/push.rocks/smarts3.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://code.foss.global/push.rocks/smarts3/issues"
|
||||
},
|
||||
"packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748",
|
||||
"pnpm": {
|
||||
"overrides": {}
|
||||
}
|
||||
}
|
||||
|
||||
8610
pnpm-lock.yaml
generated
Normal file
8610
pnpm-lock.yaml
generated
Normal file
File diff suppressed because it is too large
Load Diff
438
production-readiness.md
Normal file
438
production-readiness.md
Normal file
@@ -0,0 +1,438 @@
|
||||
# Production-Readiness Plan for smarts3
|
||||
|
||||
**Goal:** Make smarts3 production-ready as a MinIO alternative for use cases where:
|
||||
- Running MinIO is out of scope
|
||||
- You have a program written for S3 and want to use the local filesystem
|
||||
- You need a lightweight, zero-dependency S3-compatible server
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Current State Analysis
|
||||
|
||||
### ✅ What's Working
|
||||
|
||||
- **Native S3 server** with zero framework dependencies
|
||||
- **Core S3 operations:** PUT, GET, HEAD, DELETE (objects & buckets)
|
||||
- **List buckets and objects** (V1 and V2 API)
|
||||
- **Object copy** with metadata handling
|
||||
- **Range requests** for partial downloads
|
||||
- **MD5 checksums** and ETag support
|
||||
- **Custom metadata** (x-amz-meta-*)
|
||||
- **Filesystem-backed storage** with Windows compatibility
|
||||
- **S3-compatible XML error responses**
|
||||
- **Middleware system** and routing
|
||||
- **AWS SDK v3 compatibility** (tested)
|
||||
|
||||
### ❌ Production Gaps Identified
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Critical Features (Required for Production)
|
||||
|
||||
### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY**
|
||||
|
||||
**Why:** Essential for uploading files >5MB efficiently. Without this, smarts3 can't handle real-world production workloads.
|
||||
|
||||
**Implementation Required:**
|
||||
- `POST /:bucket/:key?uploads` - CreateMultipartUpload
|
||||
- `PUT /:bucket/:key?partNumber=X&uploadId=Y` - UploadPart
|
||||
- `POST /:bucket/:key?uploadId=X` - CompleteMultipartUpload
|
||||
- `DELETE /:bucket/:key?uploadId=X` - AbortMultipartUpload
|
||||
- `GET /:bucket/:key?uploadId=X` - ListParts
|
||||
- Multipart state management (temp storage for parts)
|
||||
- Part ETag tracking and validation
|
||||
- Automatic cleanup of abandoned uploads
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/multipart.controller.ts` (new)
|
||||
- `ts/classes/filesystem-store.ts` (add multipart methods)
|
||||
- `ts/classes/smarts3-server.ts` (add multipart routes)
|
||||
|
||||
---
|
||||
|
||||
### 2. Configurable Authentication 🔐
|
||||
|
||||
**Why:** Currently hardcoded credentials ('S3RVER'/'S3RVER'). Production needs custom credentials.
|
||||
|
||||
**Implementation Required:**
|
||||
- Support custom access keys and secrets via configuration
|
||||
- Implement AWS Signature V4 verification
|
||||
- Support multiple credential pairs (IAM-like users)
|
||||
- Optional: Disable authentication for local dev use
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface IAuthConfig {
|
||||
enabled: boolean;
|
||||
credentials: Array<{
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}>;
|
||||
signatureVersion: 'v4' | 'none';
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/auth-middleware.ts` (new)
|
||||
- `ts/classes/signature-validator.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate auth middleware)
|
||||
- `ts/index.ts` (add auth config options)
|
||||
|
||||
---
|
||||
|
||||
### 3. CORS Support 🌐
|
||||
|
||||
**Why:** Required for browser-based uploads and modern web apps.
|
||||
|
||||
**Implementation Required:**
|
||||
- Add CORS middleware
|
||||
- Support preflight OPTIONS requests
|
||||
- Configurable CORS origins, methods, headers
|
||||
- Per-bucket CORS configuration (optional)
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface ICorsConfig {
|
||||
enabled: boolean;
|
||||
allowedOrigins: string[]; // ['*'] or ['https://example.com']
|
||||
allowedMethods: string[]; // ['GET', 'POST', 'PUT', 'DELETE']
|
||||
allowedHeaders: string[]; // ['*'] or specific headers
|
||||
exposedHeaders: string[]; // ['ETag', 'x-amz-*']
|
||||
maxAge: number; // 3600 (seconds)
|
||||
allowCredentials: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/cors-middleware.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate CORS middleware)
|
||||
- `ts/index.ts` (add CORS config options)
|
||||
|
||||
---
|
||||
|
||||
### 4. SSL/TLS Support 🔒
|
||||
|
||||
**Why:** Production systems require encrypted connections.
|
||||
|
||||
**Implementation Required:**
|
||||
- HTTPS server option with cert/key configuration
|
||||
- Auto-redirect HTTP to HTTPS (optional)
|
||||
- Support for self-signed certs in dev mode
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface ISslConfig {
|
||||
enabled: boolean;
|
||||
cert: string; // Path to certificate file or cert content
|
||||
key: string; // Path to key file or key content
|
||||
ca?: string; // Optional CA cert
|
||||
redirectHttp?: boolean; // Redirect HTTP to HTTPS
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/smarts3-server.ts` (add HTTPS server creation)
|
||||
- `ts/index.ts` (add SSL config options)
|
||||
|
||||
---
|
||||
|
||||
### 5. Production Configuration System ⚙️
|
||||
|
||||
**Why:** Production needs flexible configuration, not just constructor options.
|
||||
|
||||
**Implementation Required:**
|
||||
- Support configuration file (JSON/YAML)
|
||||
- Environment variable support
|
||||
- Configuration validation
|
||||
- Sensible production defaults
|
||||
- Example configurations for common use cases
|
||||
|
||||
**Configuration File Example (`smarts3.config.json`):**
|
||||
```json
|
||||
{
|
||||
"server": {
|
||||
"port": 3000,
|
||||
"address": "0.0.0.0",
|
||||
"ssl": {
|
||||
"enabled": true,
|
||||
"cert": "./certs/server.crt",
|
||||
"key": "./certs/server.key"
|
||||
}
|
||||
},
|
||||
"storage": {
|
||||
"directory": "./s3-data",
|
||||
"cleanSlate": false
|
||||
},
|
||||
"auth": {
|
||||
"enabled": true,
|
||||
"credentials": [
|
||||
{
|
||||
"accessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cors": {
|
||||
"enabled": true,
|
||||
"allowedOrigins": ["*"],
|
||||
"allowedMethods": ["GET", "POST", "PUT", "DELETE", "HEAD"],
|
||||
"allowedHeaders": ["*"]
|
||||
},
|
||||
"limits": {
|
||||
"maxObjectSize": 5368709120,
|
||||
"maxMetadataSize": 2048,
|
||||
"requestTimeout": 300000
|
||||
},
|
||||
"logging": {
|
||||
"level": "info",
|
||||
"format": "json",
|
||||
"accessLog": {
|
||||
"enabled": true,
|
||||
"path": "./logs/access.log"
|
||||
},
|
||||
"errorLog": {
|
||||
"enabled": true,
|
||||
"path": "./logs/error.log"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/config-loader.ts` (new)
|
||||
- `ts/classes/config-validator.ts` (new)
|
||||
- `ts/index.ts` (use config loader)
|
||||
- Create example config files in root
|
||||
|
||||
---
|
||||
|
||||
### 6. Production Logging 📝
|
||||
|
||||
**Why:** Console logs aren't suitable for production monitoring.
|
||||
|
||||
**Implementation Required:**
|
||||
- Structured logging (JSON format option)
|
||||
- Log levels (ERROR, WARN, INFO, DEBUG)
|
||||
- File rotation support
|
||||
- Access logs (S3 standard format)
|
||||
- Integration with logging library
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?)
|
||||
- `ts/classes/access-logger-middleware.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (replace console.log with logger)
|
||||
- All controller files (use structured logging)
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Important Features (Should Have)
|
||||
|
||||
### 7. Health Check & Metrics 💊
|
||||
|
||||
**Implementation Required:**
|
||||
- `GET /_health` endpoint (non-S3, for monitoring)
|
||||
- `GET /_metrics` endpoint (Prometheus format?)
|
||||
- Server stats (requests/sec, storage used, uptime)
|
||||
- Readiness/liveness probes for Kubernetes
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/health.controller.ts` (new)
|
||||
- `ts/classes/metrics-collector.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (add health routes)
|
||||
|
||||
---
|
||||
|
||||
### 8. Batch Operations 📦
|
||||
|
||||
**Implementation Required:**
|
||||
- `POST /:bucket?delete` - DeleteObjects (delete multiple objects in one request)
|
||||
- Essential for efficient cleanup operations
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/object.controller.ts` (add deleteObjects method)
|
||||
|
||||
---
|
||||
|
||||
### 9. Request Size Limits & Validation 🛡️
|
||||
|
||||
**Implementation Required:**
|
||||
- Max object size configuration
|
||||
- Max metadata size limits
|
||||
- Request timeout configuration
|
||||
- Body size limits
|
||||
- Bucket name validation (S3 rules)
|
||||
- Key name validation
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/validation-middleware.ts` (new)
|
||||
- `ts/utils/validators.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate validation middleware)
|
||||
|
||||
---
|
||||
|
||||
### 10. Conditional Requests 🔄
|
||||
|
||||
**Implementation Required:**
|
||||
- If-Match / If-None-Match (ETag validation)
|
||||
- If-Modified-Since / If-Unmodified-Since
|
||||
- Required for caching and conflict prevention
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/object.controller.ts` (add conditional logic to GET/HEAD)
|
||||
|
||||
---
|
||||
|
||||
### 11. Graceful Shutdown 👋
|
||||
|
||||
**Implementation Required:**
|
||||
- Drain existing connections
|
||||
- Reject new connections
|
||||
- Clean multipart cleanup on shutdown
|
||||
- SIGTERM/SIGINT handling
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/smarts3-server.ts` (add graceful shutdown logic)
|
||||
- `ts/index.ts` (add signal handlers)
|
||||
|
||||
---
|
||||
|
||||
## 💡 Nice-to-Have Features
|
||||
|
||||
### 12. Advanced Features
|
||||
|
||||
- Bucket versioning support
|
||||
- Object tagging
|
||||
- Lifecycle policies (auto-delete old objects)
|
||||
- Storage class simulation (STANDARD, GLACIER, etc.)
|
||||
- Server-side encryption simulation
|
||||
- Presigned URL support (for time-limited access)
|
||||
|
||||
### 13. Performance Optimizations
|
||||
|
||||
- Stream optimization for large files
|
||||
- Optional in-memory caching for small objects
|
||||
- Parallel upload/download support
|
||||
- Compression support (gzip)
|
||||
|
||||
### 14. Developer Experience
|
||||
|
||||
- Docker image for easy deployment
|
||||
- Docker Compose examples
|
||||
- Kubernetes manifests
|
||||
- CLI for server management
|
||||
- Admin API for bucket management
|
||||
|
||||
---
|
||||
|
||||
## 📐 Implementation Phases
|
||||
|
||||
### Phase 1: Critical Production Features (Priority 1)
|
||||
|
||||
**Estimated Effort:** 2-3 weeks
|
||||
|
||||
1. ✅ Multipart uploads (biggest technical lift)
|
||||
2. ✅ Configurable authentication
|
||||
3. ✅ CORS middleware
|
||||
4. ✅ Production configuration system
|
||||
5. ✅ Production logging
|
||||
|
||||
**Outcome:** smarts3 can handle real production workloads
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Reliability & Operations (Priority 2)
|
||||
|
||||
**Estimated Effort:** 1-2 weeks
|
||||
|
||||
6. ✅ SSL/TLS support
|
||||
7. ✅ Health checks & metrics
|
||||
8. ✅ Request validation & limits
|
||||
9. ✅ Graceful shutdown
|
||||
10. ✅ Batch operations
|
||||
|
||||
**Outcome:** smarts3 is operationally mature
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: S3 Compatibility (Priority 3)
|
||||
|
||||
**Estimated Effort:** 1-2 weeks
|
||||
|
||||
11. ✅ Conditional requests
|
||||
12. ✅ Additional S3 features as needed
|
||||
13. ✅ Comprehensive test suite
|
||||
14. ✅ Documentation updates
|
||||
|
||||
**Outcome:** smarts3 has broad S3 API compatibility
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Polish (Priority 4)
|
||||
|
||||
**Estimated Effort:** As needed
|
||||
|
||||
15. ✅ Docker packaging
|
||||
16. ✅ Performance optimization
|
||||
17. ✅ Advanced features based on user feedback
|
||||
|
||||
**Outcome:** smarts3 is a complete MinIO alternative
|
||||
|
||||
---
|
||||
|
||||
## 🤔 Open Questions
|
||||
|
||||
1. **Authentication:** Do you want full AWS Signature V4 validation, or simpler static credential checking?
|
||||
2. **Configuration:** Prefer JSON, YAML, or .env file format?
|
||||
3. **Logging:** Do you have a preferred logging library, or shall I use @push.rocks/smartlog?
|
||||
4. **Scope:** Should we tackle all of Phase 1, or start with a subset (e.g., just multipart + auth)?
|
||||
5. **Testing:** Should we add comprehensive tests as we go, or batch them at the end?
|
||||
6. **Breaking changes:** Can I modify the constructor options interface, or must it remain backward compatible?
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Target Use Cases
|
||||
|
||||
**With this plan implemented, smarts3 will be a solid MinIO alternative for:**
|
||||
|
||||
✅ **Local S3 development** - Fast, simple, no Docker required
|
||||
✅ **Testing S3 integrations** - Reliable, repeatable tests
|
||||
✅ **Microservices using S3 API** with filesystem backend
|
||||
✅ **CI/CD pipelines** - Lightweight S3 for testing
|
||||
✅ **Small-to-medium production deployments** where MinIO is overkill
|
||||
✅ **Edge computing** - S3 API for local file storage
|
||||
✅ **Embedded systems** - Minimal dependencies, small footprint
|
||||
|
||||
---
|
||||
|
||||
## 📊 Current vs. Production Comparison
|
||||
|
||||
| Feature | Current | After Phase 1 | After Phase 2 | Production Ready |
|
||||
|---------|---------|---------------|---------------|------------------|
|
||||
| Basic S3 ops | ✅ | ✅ | ✅ | ✅ |
|
||||
| Multipart upload | ❌ | ✅ | ✅ | ✅ |
|
||||
| Authentication | ⚠️ (hardcoded) | ✅ | ✅ | ✅ |
|
||||
| CORS | ❌ | ✅ | ✅ | ✅ |
|
||||
| SSL/TLS | ❌ | ❌ | ✅ | ✅ |
|
||||
| Config files | ❌ | ✅ | ✅ | ✅ |
|
||||
| Production logging | ⚠️ (console) | ✅ | ✅ | ✅ |
|
||||
| Health checks | ❌ | ❌ | ✅ | ✅ |
|
||||
| Request limits | ❌ | ❌ | ✅ | ✅ |
|
||||
| Graceful shutdown | ❌ | ❌ | ✅ | ✅ |
|
||||
| Conditional requests | ❌ | ❌ | ❌ | ✅ |
|
||||
| Batch operations | ❌ | ❌ | ✅ | ✅ |
|
||||
|
||||
---
|
||||
|
||||
## 📝 Notes
|
||||
|
||||
- All features should maintain backward compatibility where possible
|
||||
- Each feature should include comprehensive tests
|
||||
- Documentation (readme.md) should be updated as features are added
|
||||
- Consider adding a migration guide for users upgrading from testing to production use
|
||||
- Performance benchmarks should be established and maintained
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** 2025-11-23
|
||||
**Status:** Planning Phase
|
||||
**Next Step:** Get approval and prioritize implementation order
|
||||
60
readme.hints.md
Normal file
60
readme.hints.md
Normal file
@@ -0,0 +1,60 @@
|
||||
# Project Hints for smarts3
|
||||
|
||||
## Current State (v6.0.0-dev)
|
||||
|
||||
- **Rust-powered S3 server** via `@push.rocks/smartrust` IPC bridge
|
||||
- High-performance: streaming I/O, zero-copy, backpressure, range seek
|
||||
- TypeScript is thin IPC wrapper; all HTTP/storage/routing in Rust binary `rusts3`
|
||||
- Full S3 compatibility: PUT, GET, HEAD, DELETE for objects and buckets
|
||||
- Multipart upload support (streaming, no OOM)
|
||||
- Authentication (AWS v2/v4 signature key extraction)
|
||||
- CORS support
|
||||
- ListBuckets, ListObjects (v1/v2), CopyObject
|
||||
|
||||
## Architecture
|
||||
|
||||
### Rust Binary (`rust/src/`)
|
||||
- `main.rs` - Clap CLI, management mode entry
|
||||
- `config.rs` - Serde config structs matching TS interfaces
|
||||
- `management.rs` - IPC loop (newline-delimited JSON over stdin/stdout)
|
||||
- `server.rs` - hyper 1.x HTTP server, routing, CORS, auth, all S3 handlers
|
||||
- `storage.rs` - FileStore: filesystem-backed storage, multipart manager
|
||||
- `xml_response.rs` - S3 XML response builders
|
||||
- `s3_error.rs` - S3 error codes with HTTP status mapping
|
||||
|
||||
### TypeScript Bridge (`ts/`)
|
||||
- `ts/index.ts` - Smarts3 class with RustBridge<TRustS3Commands>
|
||||
- `ts/plugins.ts` - path, smartpath, RustBridge, tsclass
|
||||
- `ts/paths.ts` - packageDir, bucketsDir defaults
|
||||
|
||||
### IPC Commands
|
||||
| Command | Params | Action |
|
||||
|---------|--------|--------|
|
||||
| `start` | `{ config: ISmarts3Config }` | Init storage + HTTP server |
|
||||
| `stop` | `{}` | Graceful shutdown |
|
||||
| `createBucket` | `{ name: string }` | Create bucket directory |
|
||||
|
||||
### Storage Layout (backward-compatible)
|
||||
- Objects: `{root}/{bucket}/{key}._S3_object`
|
||||
- Metadata: `{root}/{bucket}/{key}._S3_object.metadata.json`
|
||||
- MD5: `{root}/{bucket}/{key}._S3_object.md5`
|
||||
- Multipart: `{root}/.multipart/{upload_id}/part-{N}`
|
||||
|
||||
## Build
|
||||
|
||||
- `pnpm build` runs `tsrust && tsbuild --web --allowimplicitany`
|
||||
- `tsrust` compiles Rust to `dist_rust/rusts3`
|
||||
- Targets: linux_amd64, linux_arm64 (configured in npmextra.json)
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `@push.rocks/smartrust` - RustBridge IPC bridge
|
||||
- `@push.rocks/smartpath` - Path utilities
|
||||
- `@tsclass/tsclass` - IS3Descriptor type
|
||||
- `@git.zone/tsrust` (devDep) - Rust cross-compilation
|
||||
|
||||
## Testing
|
||||
|
||||
- `test/test.aws-sdk.node.ts` - AWS SDK v3 compatibility (10 tests)
|
||||
- `test/test.ts` - SmartBucket integration (3 tests)
|
||||
- Run: `pnpm test` or `tstest test/test.aws-sdk.node.ts --verbose`
|
||||
411
readme.md
411
readme.md
@@ -1,39 +1,392 @@
|
||||
# @pushrocks/smarts3
|
||||
create an s3 endpoint that maps to a local directory
|
||||
# @push.rocks/smarts3 🚀
|
||||
|
||||
## Availabililty and Links
|
||||
* [npmjs.org (npm package)](https://www.npmjs.com/package/@pushrocks/smarts3)
|
||||
* [gitlab.com (source)](https://gitlab.com/pushrocks/smarts3)
|
||||
* [github.com (source mirror)](https://github.com/pushrocks/smarts3)
|
||||
* [docs (typedoc)](https://pushrocks.gitlab.io/smarts3/)
|
||||
A high-performance, S3-compatible local server powered by a **Rust core** with a clean TypeScript API. Drop-in replacement for AWS S3 during development and testing — no cloud, no Docker, no MinIO. Just `npm install` and go.
|
||||
|
||||
## Status for master
|
||||
## Issue Reporting and Security
|
||||
|
||||
Status Category | Status Badge
|
||||
-- | --
|
||||
GitLab Pipelines | [](https://lossless.cloud)
|
||||
GitLab Pipline Test Coverage | [](https://lossless.cloud)
|
||||
npm | [](https://lossless.cloud)
|
||||
Snyk | [](https://lossless.cloud)
|
||||
TypeScript Support | [](https://lossless.cloud)
|
||||
node Support | [](https://nodejs.org/dist/latest-v10.x/docs/api/)
|
||||
Code Style | [](https://lossless.cloud)
|
||||
PackagePhobia (total standalone install weight) | [](https://lossless.cloud)
|
||||
PackagePhobia (package size on registry) | [](https://lossless.cloud)
|
||||
BundlePhobia (total size when bundled) | [](https://lossless.cloud)
|
||||
Platform support | [](https://lossless.cloud) [](https://lossless.cloud)
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
## Usage
|
||||
## 🌟 Why smarts3?
|
||||
|
||||
Use TypeScript for best in class intellisense
|
||||
| Feature | smarts3 | MinIO | s3rver |
|
||||
|---------|---------|-------|--------|
|
||||
| Install | `pnpm add` | Docker / binary | `npm install` |
|
||||
| Startup time | ~20ms | seconds | ~200ms |
|
||||
| Large file uploads | ✅ Streaming, zero-copy | ✅ | ❌ OOM risk |
|
||||
| Range requests | ✅ Seek-based | ✅ | ❌ Full read |
|
||||
| Language | Rust + TypeScript | Go | JavaScript |
|
||||
| Multipart uploads | ✅ Full support | ✅ | ❌ |
|
||||
| Auth | AWS v2/v4 key extraction | Full IAM | Basic |
|
||||
|
||||
## Contribution
|
||||
### Core Features
|
||||
|
||||
We are always happy for code contributions. If you are not the code contributing type that is ok. Still, maintaining Open Source repositories takes considerable time and thought. If you like the quality of what we do and our modules are useful to you we would appreciate a little monthly contribution: You can [contribute one time](https://lossless.link/contribute-onetime) or [contribute monthly](https://lossless.link/contribute). :)
|
||||
- ⚡ **Rust-powered HTTP server** — hyper 1.x with streaming I/O, zero-copy, backpressure
|
||||
- 🔄 **Full S3 API compatibility** — works with AWS SDK v3, SmartBucket, any S3 client
|
||||
- 📂 **Filesystem-backed storage** — buckets map to directories, objects to files
|
||||
- 📤 **Streaming multipart uploads** — large files without memory pressure
|
||||
- 🎯 **Byte-range requests** — `seek()` directly to the requested byte offset
|
||||
- 🔐 **Authentication** — AWS v2/v4 signature key extraction
|
||||
- 🌐 **CORS middleware** — configurable cross-origin support
|
||||
- 📊 **Structured logging** — tracing-based, error through debug levels
|
||||
- 🧹 **Clean slate mode** — wipe storage on startup for test isolation
|
||||
- 🧪 **Test-first design** — start/stop in milliseconds, no port conflicts
|
||||
|
||||
For further information read the linked docs at the top of this readme.
|
||||
## 📦 Installation
|
||||
|
||||
> MIT licensed | **©** [Lossless GmbH](https://lossless.gmbh)
|
||||
| By using this npm module you agree to our [privacy policy](https://lossless.gmbH/privacy)
|
||||
```bash
|
||||
pnpm add @push.rocks/smarts3 -D
|
||||
```
|
||||
|
||||
[](https://maintainedby.lossless.com)
|
||||
> **Note:** The package ships with precompiled Rust binaries for `linux_amd64` and `linux_arm64`. No Rust toolchain needed on your machine.
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
|
||||
// Start a local S3 server
|
||||
const s3 = await Smarts3.createAndStart({
|
||||
server: { port: 3000 },
|
||||
storage: { cleanSlate: true },
|
||||
});
|
||||
|
||||
// Create a bucket
|
||||
await s3.createBucket('my-bucket');
|
||||
|
||||
// Get connection details for any S3 client
|
||||
const descriptor = await s3.getS3Descriptor();
|
||||
// → { endpoint: 'localhost', port: 3000, accessKey: 'S3RVER', accessSecret: 'S3RVER', useSsl: false }
|
||||
|
||||
// When done
|
||||
await s3.stop();
|
||||
```
|
||||
|
||||
## 📖 Configuration
|
||||
|
||||
All config fields are optional — sensible defaults are applied automatically.
|
||||
|
||||
```typescript
|
||||
import { Smarts3, ISmarts3Config } from '@push.rocks/smarts3';
|
||||
|
||||
const config: ISmarts3Config = {
|
||||
server: {
|
||||
port: 3000, // Default: 3000
|
||||
address: '0.0.0.0', // Default: '0.0.0.0'
|
||||
silent: false, // Default: false
|
||||
},
|
||||
storage: {
|
||||
directory: './my-data', // Default: .nogit/bucketsDir
|
||||
cleanSlate: false, // Default: false — set true to wipe on start
|
||||
},
|
||||
auth: {
|
||||
enabled: false, // Default: false
|
||||
credentials: [{
|
||||
accessKeyId: 'MY_KEY',
|
||||
secretAccessKey: 'MY_SECRET',
|
||||
}],
|
||||
},
|
||||
cors: {
|
||||
enabled: false, // Default: false
|
||||
allowedOrigins: ['*'],
|
||||
allowedMethods: ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'],
|
||||
allowedHeaders: ['*'],
|
||||
exposedHeaders: ['ETag', 'x-amz-request-id', 'x-amz-version-id'],
|
||||
maxAge: 86400,
|
||||
allowCredentials: false,
|
||||
},
|
||||
logging: {
|
||||
level: 'info', // 'error' | 'warn' | 'info' | 'debug'
|
||||
format: 'text', // 'text' | 'json'
|
||||
enabled: true,
|
||||
},
|
||||
limits: {
|
||||
maxObjectSize: 5 * 1024 * 1024 * 1024, // 5 GB
|
||||
maxMetadataSize: 2048,
|
||||
requestTimeout: 300000, // 5 minutes
|
||||
},
|
||||
multipart: {
|
||||
expirationDays: 7,
|
||||
cleanupIntervalMinutes: 60,
|
||||
},
|
||||
};
|
||||
|
||||
const s3 = await Smarts3.createAndStart(config);
|
||||
```
|
||||
|
||||
### Common Configurations
|
||||
|
||||
**CI/CD testing** — silent, clean, fast:
|
||||
```typescript
|
||||
const s3 = await Smarts3.createAndStart({
|
||||
server: { port: 9999, silent: true },
|
||||
storage: { cleanSlate: true },
|
||||
});
|
||||
```
|
||||
|
||||
**Auth enabled:**
|
||||
```typescript
|
||||
const s3 = await Smarts3.createAndStart({
|
||||
auth: {
|
||||
enabled: true,
|
||||
credentials: [{ accessKeyId: 'test', secretAccessKey: 'test123' }],
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
**CORS for local web dev:**
|
||||
```typescript
|
||||
const s3 = await Smarts3.createAndStart({
|
||||
cors: {
|
||||
enabled: true,
|
||||
allowedOrigins: ['http://localhost:5173'],
|
||||
allowCredentials: true,
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## 📤 Usage with AWS SDK v3
|
||||
|
||||
```typescript
|
||||
import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
||||
|
||||
const descriptor = await s3.getS3Descriptor();
|
||||
|
||||
const client = new S3Client({
|
||||
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: descriptor.accessKey,
|
||||
secretAccessKey: descriptor.accessSecret,
|
||||
},
|
||||
forcePathStyle: true, // Required for path-style S3
|
||||
});
|
||||
|
||||
// Upload
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'hello.txt',
|
||||
Body: 'Hello, S3!',
|
||||
ContentType: 'text/plain',
|
||||
}));
|
||||
|
||||
// Download
|
||||
const { Body } = await client.send(new GetObjectCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'hello.txt',
|
||||
}));
|
||||
const content = await Body.transformToString(); // "Hello, S3!"
|
||||
|
||||
// Delete
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'hello.txt',
|
||||
}));
|
||||
```
|
||||
|
||||
## 🪣 Usage with SmartBucket
|
||||
|
||||
```typescript
|
||||
import { SmartBucket } from '@push.rocks/smartbucket';
|
||||
|
||||
const smartbucket = new SmartBucket(await s3.getS3Descriptor());
|
||||
const bucket = await smartbucket.createBucket('my-bucket');
|
||||
const dir = await bucket.getBaseDirectory();
|
||||
|
||||
// Upload
|
||||
await dir.fastPut({ path: 'docs/readme.txt', contents: 'Hello!' });
|
||||
|
||||
// Download
|
||||
const content = await dir.fastGet('docs/readme.txt');
|
||||
|
||||
// List
|
||||
const files = await dir.listFiles();
|
||||
```
|
||||
|
||||
## 📤 Multipart Uploads
|
||||
|
||||
For files larger than 5 MB, use multipart uploads. smarts3 handles them with **streaming I/O** — parts are written directly to disk, never buffered in memory.
|
||||
|
||||
```typescript
|
||||
import {
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
|
||||
// 1. Initiate
|
||||
const { UploadId } = await client.send(new CreateMultipartUploadCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'large-file.bin',
|
||||
}));
|
||||
|
||||
// 2. Upload parts
|
||||
const parts = [];
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const { ETag } = await client.send(new UploadPartCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'large-file.bin',
|
||||
UploadId,
|
||||
PartNumber: i + 1,
|
||||
Body: chunks[i],
|
||||
}));
|
||||
parts.push({ PartNumber: i + 1, ETag });
|
||||
}
|
||||
|
||||
// 3. Complete
|
||||
await client.send(new CompleteMultipartUploadCommand({
|
||||
Bucket: 'my-bucket',
|
||||
Key: 'large-file.bin',
|
||||
UploadId,
|
||||
MultipartUpload: { Parts: parts },
|
||||
}));
|
||||
```
|
||||
|
||||
## 🧪 Testing Integration
|
||||
|
||||
```typescript
|
||||
import { Smarts3 } from '@push.rocks/smarts3';
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
|
||||
let s3: Smarts3;
|
||||
|
||||
tap.test('setup', async () => {
|
||||
s3 = await Smarts3.createAndStart({
|
||||
server: { port: 4567, silent: true },
|
||||
storage: { cleanSlate: true },
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('should store and retrieve objects', async () => {
|
||||
await s3.createBucket('test');
|
||||
// ... your test logic using AWS SDK or SmartBucket
|
||||
});
|
||||
|
||||
tap.test('teardown', async () => {
|
||||
await s3.stop();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
```
|
||||
|
||||
## 🔧 API Reference
|
||||
|
||||
### `Smarts3` Class
|
||||
|
||||
#### `static createAndStart(config?: ISmarts3Config): Promise<Smarts3>`
|
||||
|
||||
Create and start a server in one call.
|
||||
|
||||
#### `start(): Promise<void>`
|
||||
|
||||
Spawn the Rust binary and start the HTTP server.
|
||||
|
||||
#### `stop(): Promise<void>`
|
||||
|
||||
Gracefully stop the server and kill the Rust process.
|
||||
|
||||
#### `createBucket(name: string): Promise<{ name: string }>`
|
||||
|
||||
Create an S3 bucket.
|
||||
|
||||
#### `getS3Descriptor(options?): Promise<IS3Descriptor>`
|
||||
|
||||
Get connection details for S3 clients. Returns:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `endpoint` | `string` | Server hostname (`localhost` by default) |
|
||||
| `port` | `number` | Server port |
|
||||
| `accessKey` | `string` | Access key from first configured credential |
|
||||
| `accessSecret` | `string` | Secret key from first configured credential |
|
||||
| `useSsl` | `boolean` | Always `false` (plain HTTP) |
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
smarts3 uses a **hybrid Rust + TypeScript** architecture:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
│ Your Code (AWS SDK, etc.) │
|
||||
│ ↕ HTTP (localhost:3000) │
|
||||
├─────────────────────────────────┤
|
||||
│ rusts3 binary (Rust) │
|
||||
│ ├─ hyper 1.x HTTP server │
|
||||
│ ├─ S3 path-style routing │
|
||||
│ ├─ Streaming storage layer │
|
||||
│ ├─ Multipart manager │
|
||||
│ ├─ CORS / Auth middleware │
|
||||
│ └─ S3 XML response builder │
|
||||
├─────────────────────────────────┤
|
||||
│ TypeScript (thin IPC wrapper) │
|
||||
│ ├─ Smarts3 class │
|
||||
│ ├─ RustBridge (stdin/stdout) │
|
||||
│ └─ Config & S3 descriptor │
|
||||
└─────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Why Rust?** The TypeScript implementation had critical perf issues: OOM on multipart uploads (parts buffered in memory), double stream copying, file descriptor leaks on HEAD requests, full-file reads for range requests, and no backpressure. The Rust binary solves all of these with streaming I/O, zero-copy, and direct `seek()` for range requests.
|
||||
|
||||
**IPC Protocol:** TypeScript spawns the `rusts3` binary with `--management` and communicates via newline-delimited JSON over stdin/stdout. Commands: `start`, `stop`, `createBucket`.
|
||||
|
||||
### S3 Operations Supported
|
||||
|
||||
| Operation | Method | Path |
|
||||
|-----------|--------|------|
|
||||
| ListBuckets | `GET /` | |
|
||||
| CreateBucket | `PUT /{bucket}` | |
|
||||
| DeleteBucket | `DELETE /{bucket}` | |
|
||||
| HeadBucket | `HEAD /{bucket}` | |
|
||||
| ListObjects (v1/v2) | `GET /{bucket}` | `?list-type=2` for v2 |
|
||||
| PutObject | `PUT /{bucket}/{key}` | |
|
||||
| GetObject | `GET /{bucket}/{key}` | Supports `Range` header |
|
||||
| HeadObject | `HEAD /{bucket}/{key}` | |
|
||||
| DeleteObject | `DELETE /{bucket}/{key}` | |
|
||||
| CopyObject | `PUT /{bucket}/{key}` | `x-amz-copy-source` header |
|
||||
| InitiateMultipartUpload | `POST /{bucket}/{key}?uploads` | |
|
||||
| UploadPart | `PUT /{bucket}/{key}?partNumber&uploadId` | |
|
||||
| CompleteMultipartUpload | `POST /{bucket}/{key}?uploadId` | |
|
||||
| AbortMultipartUpload | `DELETE /{bucket}/{key}?uploadId` | |
|
||||
| ListMultipartUploads | `GET /{bucket}?uploads` | |
|
||||
|
||||
### On-Disk Format
|
||||
|
||||
```
|
||||
{storage.directory}/
|
||||
{bucket}/
|
||||
{key}._S3_object # Object data
|
||||
{key}._S3_object.metadata.json # Metadata (content-type, x-amz-meta-*, etc.)
|
||||
{key}._S3_object.md5 # Cached MD5 hash
|
||||
.multipart/
|
||||
{upload-id}/
|
||||
metadata.json # Upload metadata (bucket, key, parts)
|
||||
part-1 # Part data files
|
||||
part-2
|
||||
...
|
||||
```
|
||||
|
||||
## 🔗 Related Packages
|
||||
|
||||
- [`@push.rocks/smartbucket`](https://code.foss.global/push.rocks/smartbucket) — High-level S3 abstraction layer
|
||||
- [`@push.rocks/smartrust`](https://code.foss.global/push.rocks/smartrust) — TypeScript ↔ Rust IPC bridge
|
||||
- [`@git.zone/tsrust`](https://code.foss.global/git.zone/tsrust) — Rust cross-compilation for npm packages
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
||||
|
||||
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District Court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
2
rust/.cargo/config.toml
Normal file
2
rust/.cargo/config.toml
Normal file
@@ -0,0 +1,2 @@
|
||||
[target.aarch64-unknown-linux-gnu]
|
||||
linker = "aarch64-linux-gnu-gcc"
|
||||
1393
rust/Cargo.lock
generated
Normal file
1393
rust/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
rust/Cargo.toml
Normal file
30
rust/Cargo.toml
Normal file
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "rusts3"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "rusts3"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
hyper = { version = "1", features = ["http1", "server"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "http1"] }
|
||||
http-body-util = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
quick-xml = { version = "0.37", features = ["serialize"] }
|
||||
md-5 = "0.10"
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
bytes = "1"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
percent-encoding = "2"
|
||||
url = "2"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
futures-core = "0.3"
|
||||
78
rust/src/config.rs
Normal file
78
rust/src/config.rs
Normal file
@@ -0,0 +1,78 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct S3Config {
|
||||
pub server: ServerConfig,
|
||||
pub storage: StorageConfig,
|
||||
pub auth: AuthConfig,
|
||||
pub cors: CorsConfig,
|
||||
pub logging: LoggingConfig,
|
||||
pub limits: LimitsConfig,
|
||||
pub multipart: MultipartConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ServerConfig {
|
||||
pub port: u16,
|
||||
pub address: String,
|
||||
pub silent: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StorageConfig {
|
||||
pub directory: String,
|
||||
pub clean_slate: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AuthConfig {
|
||||
pub enabled: bool,
|
||||
pub credentials: Vec<Credential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Credential {
|
||||
#[serde(rename = "accessKeyId")]
|
||||
pub access_key_id: String,
|
||||
#[serde(rename = "secretAccessKey")]
|
||||
pub secret_access_key: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CorsConfig {
|
||||
pub enabled: bool,
|
||||
pub allowed_origins: Option<Vec<String>>,
|
||||
pub allowed_methods: Option<Vec<String>>,
|
||||
pub allowed_headers: Option<Vec<String>>,
|
||||
pub exposed_headers: Option<Vec<String>>,
|
||||
pub max_age: Option<u64>,
|
||||
pub allow_credentials: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LoggingConfig {
|
||||
pub level: Option<String>,
|
||||
pub format: Option<String>,
|
||||
pub enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct LimitsConfig {
|
||||
pub max_object_size: Option<u64>,
|
||||
pub max_metadata_size: Option<u64>,
|
||||
pub request_timeout: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MultipartConfig {
|
||||
pub expiration_days: Option<u64>,
|
||||
pub cleanup_interval_minutes: Option<u64>,
|
||||
}
|
||||
43
rust/src/main.rs
Normal file
43
rust/src/main.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
mod config;
|
||||
mod management;
|
||||
mod s3_error;
|
||||
mod server;
|
||||
mod storage;
|
||||
mod xml_response;
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "rusts3", about = "High-performance S3-compatible server")]
|
||||
struct Cli {
|
||||
/// Run in management mode (IPC via stdin/stdout)
|
||||
#[arg(long)]
|
||||
management: bool,
|
||||
|
||||
/// Log level
|
||||
#[arg(long, default_value = "info")]
|
||||
log_level: String,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
if cli.management {
|
||||
// Init tracing to stderr only (stdout reserved for IPC)
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(std::io::stderr)
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_new(&cli.log_level)
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
management::management_loop().await?;
|
||||
} else {
|
||||
eprintln!("rusts3: use --management flag for IPC mode");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
155
rust/src/management.rs
Normal file
155
rust/src/management.rs
Normal file
@@ -0,0 +1,155 @@
|
||||
use anyhow::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::io::Write;
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
|
||||
use crate::config::S3Config;
|
||||
use crate::server::S3Server;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct IpcRequest {
|
||||
id: String,
|
||||
method: String,
|
||||
params: Value,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct IpcResponse {
|
||||
id: String,
|
||||
success: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
result: Option<Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct IpcEvent {
|
||||
event: String,
|
||||
data: Value,
|
||||
}
|
||||
|
||||
fn send_line(value: &impl Serialize) {
|
||||
let mut stdout = std::io::stdout().lock();
|
||||
serde_json::to_writer(&mut stdout, value).ok();
|
||||
stdout.write_all(b"\n").ok();
|
||||
stdout.flush().ok();
|
||||
}
|
||||
|
||||
fn send_response(id: String, result: Value) {
|
||||
send_line(&IpcResponse {
|
||||
id,
|
||||
success: true,
|
||||
result: Some(result),
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
|
||||
fn send_error(id: String, message: String) {
|
||||
send_line(&IpcResponse {
|
||||
id,
|
||||
success: false,
|
||||
result: None,
|
||||
error: Some(message),
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn management_loop() -> Result<()> {
|
||||
// Emit ready event
|
||||
send_line(&IpcEvent {
|
||||
event: "ready".to_string(),
|
||||
data: serde_json::json!({}),
|
||||
});
|
||||
|
||||
let mut server: Option<S3Server> = None;
|
||||
let stdin = BufReader::new(tokio::io::stdin());
|
||||
let mut lines = stdin.lines();
|
||||
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
let line = line.trim().to_string();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let req: IpcRequest = match serde_json::from_str(&line) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
tracing::warn!("Invalid IPC request: {}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let id = req.id.clone();
|
||||
let method = req.method.as_str();
|
||||
|
||||
match method {
|
||||
"start" => {
|
||||
#[derive(Deserialize)]
|
||||
struct StartParams {
|
||||
config: S3Config,
|
||||
}
|
||||
match serde_json::from_value::<StartParams>(req.params) {
|
||||
Ok(params) => {
|
||||
match S3Server::start(params.config).await {
|
||||
Ok(s) => {
|
||||
server = Some(s);
|
||||
send_response(id, serde_json::json!({}));
|
||||
}
|
||||
Err(e) => {
|
||||
send_error(id, format!("Failed to start server: {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
send_error(id, format!("Invalid start params: {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
"stop" => {
|
||||
if let Some(s) = server.take() {
|
||||
s.stop().await;
|
||||
}
|
||||
send_response(id, serde_json::json!({}));
|
||||
}
|
||||
"createBucket" => {
|
||||
#[derive(Deserialize)]
|
||||
struct CreateBucketParams {
|
||||
name: String,
|
||||
}
|
||||
match serde_json::from_value::<CreateBucketParams>(req.params) {
|
||||
Ok(params) => {
|
||||
if let Some(ref s) = server {
|
||||
match s.store().create_bucket(¶ms.name).await {
|
||||
Ok(()) => {
|
||||
send_response(id, serde_json::json!({}));
|
||||
}
|
||||
Err(e) => {
|
||||
send_error(
|
||||
id,
|
||||
format!("Failed to create bucket: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
send_error(id, "Server not started".to_string());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
send_error(id, format!("Invalid createBucket params: {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
send_error(id, format!("Unknown method: {}", method));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean shutdown
|
||||
if let Some(s) = server.take() {
|
||||
s.stop().await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
70
rust/src/s3_error.rs
Normal file
70
rust/src/s3_error.rs
Normal file
@@ -0,0 +1,70 @@
|
||||
use hyper::{Response, StatusCode};
|
||||
use http_body_util::Full;
|
||||
use bytes::Bytes;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("S3Error({code}): {message}")]
|
||||
pub struct S3Error {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub status: StatusCode,
|
||||
}
|
||||
|
||||
impl S3Error {
|
||||
pub fn new(code: &str, message: &str, status: StatusCode) -> Self {
|
||||
Self {
|
||||
code: code.to_string(),
|
||||
message: message.to_string(),
|
||||
status,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn no_such_key() -> Self {
|
||||
Self::new("NoSuchKey", "The specified key does not exist.", StatusCode::NOT_FOUND)
|
||||
}
|
||||
|
||||
pub fn no_such_bucket() -> Self {
|
||||
Self::new("NoSuchBucket", "The specified bucket does not exist", StatusCode::NOT_FOUND)
|
||||
}
|
||||
|
||||
pub fn bucket_not_empty() -> Self {
|
||||
Self::new("BucketNotEmpty", "The bucket you tried to delete is not empty", StatusCode::CONFLICT)
|
||||
}
|
||||
|
||||
pub fn access_denied() -> Self {
|
||||
Self::new("AccessDenied", "Access Denied", StatusCode::FORBIDDEN)
|
||||
}
|
||||
|
||||
pub fn no_such_upload() -> Self {
|
||||
Self::new("NoSuchUpload", "The specified upload does not exist", StatusCode::NOT_FOUND)
|
||||
}
|
||||
|
||||
pub fn invalid_part_number() -> Self {
|
||||
Self::new("InvalidPartNumber", "Part number must be between 1 and 10000", StatusCode::BAD_REQUEST)
|
||||
}
|
||||
|
||||
pub fn internal_error(msg: &str) -> Self {
|
||||
Self::new("InternalError", msg, StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
|
||||
pub fn invalid_request(msg: &str) -> Self {
|
||||
Self::new("InvalidRequest", msg, StatusCode::BAD_REQUEST)
|
||||
}
|
||||
|
||||
pub fn to_xml(&self) -> String {
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<Error><Code>{}</Code><Message>{}</Message></Error>",
|
||||
self.code, self.message
|
||||
)
|
||||
}
|
||||
|
||||
pub fn to_response(&self, request_id: &str) -> Response<Full<Bytes>> {
|
||||
let xml = self.to_xml();
|
||||
Response::builder()
|
||||
.status(self.status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(Full::new(Bytes::from(xml)))
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
865
rust/src/server.rs
Normal file
865
rust/src/server.rs
Normal file
@@ -0,0 +1,865 @@
|
||||
use anyhow::Result;
|
||||
use bytes::Bytes;
|
||||
use futures_core::Stream;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Incoming;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Method, Request, Response, StatusCode};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::watch;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::S3Config;
|
||||
use crate::s3_error::S3Error;
|
||||
use crate::storage::FileStore;
|
||||
use crate::xml_response;
|
||||
|
||||
pub struct S3Server {
|
||||
store: Arc<FileStore>,
|
||||
config: S3Config,
|
||||
shutdown_tx: watch::Sender<bool>,
|
||||
server_handle: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl S3Server {
|
||||
pub async fn start(config: S3Config) -> Result<Self> {
|
||||
let store = Arc::new(FileStore::new(config.storage.directory.clone().into()));
|
||||
|
||||
// Initialize or reset storage
|
||||
if config.storage.clean_slate {
|
||||
store.reset().await?;
|
||||
} else {
|
||||
store.initialize().await?;
|
||||
}
|
||||
|
||||
let addr: SocketAddr = format!("{}:{}", config.address(), config.server.port)
|
||||
.parse()?;
|
||||
|
||||
let listener = TcpListener::bind(addr).await?;
|
||||
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
||||
|
||||
let server_store = store.clone();
|
||||
let server_config = config.clone();
|
||||
|
||||
let server_handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let mut rx = shutdown_rx.clone();
|
||||
|
||||
tokio::select! {
|
||||
result = listener.accept() => {
|
||||
match result {
|
||||
Ok((stream, _remote_addr)) => {
|
||||
let io = TokioIo::new(stream);
|
||||
let store = server_store.clone();
|
||||
let cfg = server_config.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let svc = service_fn(move |req: Request<Incoming>| {
|
||||
let store = store.clone();
|
||||
let cfg = cfg.clone();
|
||||
async move {
|
||||
handle_request(req, store, cfg).await
|
||||
}
|
||||
});
|
||||
|
||||
if let Err(e) = http1::Builder::new()
|
||||
.keep_alive(true)
|
||||
.serve_connection(io, svc)
|
||||
.await
|
||||
{
|
||||
if !e.is_incomplete_message() {
|
||||
tracing::error!("Connection error: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Accept error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = rx.changed() => {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if !config.server.silent {
|
||||
tracing::info!("S3 server listening on {}", addr);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
store,
|
||||
config,
|
||||
shutdown_tx,
|
||||
server_handle,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn stop(self) {
|
||||
let _ = self.shutdown_tx.send(true);
|
||||
let _ = self.server_handle.await;
|
||||
}
|
||||
|
||||
pub fn store(&self) -> &FileStore {
|
||||
&self.store
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Config {
|
||||
fn address(&self) -> &str {
|
||||
&self.server.address
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Request handling
|
||||
// ============================
|
||||
|
||||
type BoxBody = http_body_util::combinators::BoxBody<Bytes, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
fn full_body(data: impl Into<Bytes>) -> BoxBody {
|
||||
http_body_util::Full::new(data.into())
|
||||
.map_err(|never: std::convert::Infallible| -> Box<dyn std::error::Error + Send + Sync> { match never {} })
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn empty_body() -> BoxBody {
|
||||
http_body_util::Empty::new()
|
||||
.map_err(|never: std::convert::Infallible| -> Box<dyn std::error::Error + Send + Sync> { match never {} })
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn stream_body(reader: tokio::fs::File, content_length: u64) -> BoxBody {
|
||||
let stream = ReaderStream::with_capacity(reader.take(content_length), 64 * 1024);
|
||||
let mapped = FrameStream { inner: stream };
|
||||
http_body_util::StreamBody::new(mapped).boxed()
|
||||
}
|
||||
|
||||
/// Adapter that converts ReaderStream into a Stream of Frame<Bytes>
|
||||
struct FrameStream {
|
||||
inner: ReaderStream<tokio::io::Take<tokio::fs::File>>,
|
||||
}
|
||||
|
||||
impl Stream for FrameStream {
|
||||
type Item = Result<hyper::body::Frame<Bytes>, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let inner = unsafe { self.map_unchecked_mut(|s| &mut s.inner) };
|
||||
match inner.poll_next(cx) {
|
||||
Poll::Ready(Some(Ok(bytes))) => {
|
||||
Poll::Ready(Some(Ok(hyper::body::Frame::data(bytes))))
|
||||
}
|
||||
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>))),
|
||||
Poll::Ready(None) => Poll::Ready(None),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn xml_response(status: StatusCode, xml: String, request_id: &str) -> Response<BoxBody> {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(full_body(xml))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn empty_response(status: StatusCode, request_id: &str) -> Response<BoxBody> {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(empty_body())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn s3_error_response(err: &S3Error, request_id: &str) -> Response<BoxBody> {
|
||||
let xml = err.to_xml();
|
||||
Response::builder()
|
||||
.status(err.status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(full_body(xml))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn handle_request(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
config: S3Config,
|
||||
) -> Result<Response<BoxBody>, std::convert::Infallible> {
|
||||
let request_id = Uuid::new_v4().to_string();
|
||||
let method = req.method().clone();
|
||||
let uri = req.uri().clone();
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Handle CORS preflight
|
||||
if config.cors.enabled && method == Method::OPTIONS {
|
||||
let resp = build_cors_preflight(&config, &request_id);
|
||||
return Ok(resp);
|
||||
}
|
||||
|
||||
// Auth check
|
||||
if config.auth.enabled {
|
||||
if let Err(e) = check_auth(&req, &config) {
|
||||
tracing::warn!("Auth failed: {}", e.message);
|
||||
return Ok(s3_error_response(&e, &request_id));
|
||||
}
|
||||
}
|
||||
|
||||
// Route and handle
|
||||
let mut response = match route_request(req, store, &config, &request_id).await {
|
||||
Ok(resp) => resp,
|
||||
Err(err) => {
|
||||
if let Some(s3err) = err.downcast_ref::<S3Error>() {
|
||||
s3_error_response(s3err, &request_id)
|
||||
} else {
|
||||
tracing::error!("Internal error: {}", err);
|
||||
let s3err = S3Error::internal_error(&err.to_string());
|
||||
s3_error_response(&s3err, &request_id)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Add CORS headers if enabled
|
||||
if config.cors.enabled {
|
||||
add_cors_headers(response.headers_mut(), &config);
|
||||
}
|
||||
|
||||
let duration = start.elapsed();
|
||||
tracing::info!(
|
||||
method = %method,
|
||||
path = %uri.path(),
|
||||
status = %response.status().as_u16(),
|
||||
duration_ms = %duration.as_millis(),
|
||||
"request"
|
||||
);
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Routing
|
||||
// ============================
|
||||
|
||||
async fn route_request(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
_config: &S3Config,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let method = req.method().clone();
|
||||
let path = req.uri().path().to_string();
|
||||
let query_string = req.uri().query().unwrap_or("").to_string();
|
||||
let query = parse_query(&query_string);
|
||||
|
||||
// Parse path: /, /{bucket}, /{bucket}/{key...}
|
||||
let segments: Vec<&str> = path
|
||||
.trim_start_matches('/')
|
||||
.splitn(2, '/')
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect();
|
||||
|
||||
match segments.len() {
|
||||
0 => {
|
||||
// Root: GET / -> ListBuckets
|
||||
match method {
|
||||
Method::GET => handle_list_buckets(store, request_id).await,
|
||||
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
// Bucket level: /{bucket}
|
||||
let bucket = percent_decode(segments[0]);
|
||||
match method {
|
||||
Method::GET => {
|
||||
if query.contains_key("uploads") {
|
||||
handle_list_multipart_uploads(store, &bucket, request_id).await
|
||||
} else {
|
||||
handle_list_objects(store, &bucket, &query, request_id).await
|
||||
}
|
||||
}
|
||||
Method::PUT => handle_create_bucket(store, &bucket, request_id).await,
|
||||
Method::DELETE => handle_delete_bucket(store, &bucket, request_id).await,
|
||||
Method::HEAD => handle_head_bucket(store, &bucket, request_id).await,
|
||||
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
// Object level: /{bucket}/{key...}
|
||||
let bucket = percent_decode(segments[0]);
|
||||
let key = percent_decode(segments[1]);
|
||||
|
||||
match method {
|
||||
Method::PUT => {
|
||||
if query.contains_key("partNumber") && query.contains_key("uploadId") {
|
||||
handle_upload_part(req, store, &query, request_id).await
|
||||
} else if req.headers().contains_key("x-amz-copy-source") {
|
||||
handle_copy_object(req, store, &bucket, &key, request_id).await
|
||||
} else {
|
||||
handle_put_object(req, store, &bucket, &key, request_id).await
|
||||
}
|
||||
}
|
||||
Method::GET => {
|
||||
handle_get_object(req, store, &bucket, &key, request_id).await
|
||||
}
|
||||
Method::HEAD => {
|
||||
handle_head_object(store, &bucket, &key, request_id).await
|
||||
}
|
||||
Method::DELETE => {
|
||||
if query.contains_key("uploadId") {
|
||||
let upload_id = query.get("uploadId").unwrap();
|
||||
handle_abort_multipart(store, upload_id, request_id).await
|
||||
} else {
|
||||
handle_delete_object(store, &bucket, &key, request_id).await
|
||||
}
|
||||
}
|
||||
Method::POST => {
|
||||
if query.contains_key("uploads") {
|
||||
handle_initiate_multipart(req, store, &bucket, &key, request_id).await
|
||||
} else if query.contains_key("uploadId") {
|
||||
let upload_id = query.get("uploadId").unwrap().clone();
|
||||
handle_complete_multipart(req, store, &bucket, &key, &upload_id, request_id).await
|
||||
} else {
|
||||
let err = S3Error::invalid_request("Invalid POST request");
|
||||
Ok(s3_error_response(&err, request_id))
|
||||
}
|
||||
}
|
||||
_ => Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED, request_id)),
|
||||
}
|
||||
}
|
||||
_ => Ok(empty_response(StatusCode::BAD_REQUEST, request_id)),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Handlers
|
||||
// ============================
|
||||
|
||||
async fn handle_list_buckets(
|
||||
store: Arc<FileStore>,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let buckets = store.list_buckets().await?;
|
||||
let xml = xml_response::list_buckets_xml(&buckets);
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
async fn handle_create_bucket(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
store.create_bucket(bucket).await?;
|
||||
Ok(empty_response(StatusCode::OK, request_id))
|
||||
}
|
||||
|
||||
async fn handle_delete_bucket(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
store.delete_bucket(bucket).await?;
|
||||
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||
}
|
||||
|
||||
async fn handle_head_bucket(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
if store.bucket_exists(bucket).await {
|
||||
Ok(empty_response(StatusCode::OK, request_id))
|
||||
} else {
|
||||
Err(S3Error::no_such_bucket().into())
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_list_objects(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
query: &HashMap<String, String>,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let prefix = query.get("prefix").map(|s| s.as_str()).unwrap_or("");
|
||||
let delimiter = query.get("delimiter").map(|s| s.as_str()).unwrap_or("");
|
||||
let max_keys = query
|
||||
.get("max-keys")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(1000usize);
|
||||
let continuation_token = query.get("continuation-token").map(|s| s.as_str());
|
||||
let is_v2 = query.get("list-type").map(|s| s.as_str()) == Some("2");
|
||||
|
||||
let result = store
|
||||
.list_objects(bucket, prefix, delimiter, max_keys, continuation_token)
|
||||
.await?;
|
||||
|
||||
let xml = if is_v2 {
|
||||
xml_response::list_objects_v2_xml(bucket, &result)
|
||||
} else {
|
||||
xml_response::list_objects_v1_xml(bucket, &result)
|
||||
};
|
||||
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
async fn handle_put_object(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let metadata = extract_metadata(req.headers());
|
||||
let body = req.into_body();
|
||||
|
||||
let result = store.put_object(bucket, key, body, metadata).await?;
|
||||
|
||||
let resp = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("ETag", format!("\"{}\"", result.md5))
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(empty_body())
|
||||
.unwrap();
|
||||
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
async fn handle_get_object(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
// Parse Range header
|
||||
let range = parse_range_header(req.headers());
|
||||
|
||||
let result = store.get_object(bucket, key, range).await?;
|
||||
|
||||
let content_type = result
|
||||
.metadata
|
||||
.get("content-type")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "binary/octet-stream".to_string());
|
||||
|
||||
let mut builder = Response::builder()
|
||||
.header("ETag", format!("\"{}\"", result.md5))
|
||||
.header("Last-Modified", result.last_modified.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("Content-Type", &content_type)
|
||||
.header("Accept-Ranges", "bytes")
|
||||
.header("x-amz-request-id", request_id);
|
||||
|
||||
// Add custom metadata headers
|
||||
for (k, v) in &result.metadata {
|
||||
if k.starts_with("x-amz-meta-") {
|
||||
builder = builder.header(k.as_str(), v.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((start, end)) = range {
|
||||
let content_length = end - start + 1;
|
||||
let resp = builder
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header("Content-Length", content_length.to_string())
|
||||
.header(
|
||||
"Content-Range",
|
||||
format!("bytes {}-{}/{}", start, end, result.size),
|
||||
)
|
||||
.body(stream_body(result.body, content_length))
|
||||
.unwrap();
|
||||
Ok(resp)
|
||||
} else {
|
||||
let resp = builder
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Length", result.size.to_string())
|
||||
.body(stream_body(result.body, result.content_length))
|
||||
.unwrap();
|
||||
Ok(resp)
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_head_object(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let result = store.head_object(bucket, key).await?;
|
||||
|
||||
let content_type = result
|
||||
.metadata
|
||||
.get("content-type")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "binary/octet-stream".to_string());
|
||||
|
||||
let mut builder = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("ETag", format!("\"{}\"", result.md5))
|
||||
.header("Last-Modified", result.last_modified.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("Content-Type", &content_type)
|
||||
.header("Content-Length", result.size.to_string())
|
||||
.header("Accept-Ranges", "bytes")
|
||||
.header("x-amz-request-id", request_id);
|
||||
|
||||
for (k, v) in &result.metadata {
|
||||
if k.starts_with("x-amz-meta-") {
|
||||
builder = builder.header(k.as_str(), v.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(builder.body(empty_body()).unwrap())
|
||||
}
|
||||
|
||||
async fn handle_delete_object(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
store.delete_object(bucket, key).await?;
|
||||
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||
}
|
||||
|
||||
async fn handle_copy_object(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
dest_bucket: &str,
|
||||
dest_key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let copy_source = req
|
||||
.headers()
|
||||
.get("x-amz-copy-source")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let metadata_directive = req
|
||||
.headers()
|
||||
.get("x-amz-metadata-directive")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("COPY")
|
||||
.to_uppercase();
|
||||
|
||||
// Parse source: /bucket/key or bucket/key
|
||||
let source = copy_source.trim_start_matches('/');
|
||||
let first_slash = source.find('/').unwrap_or(source.len());
|
||||
let src_bucket = percent_decode(&source[..first_slash]);
|
||||
let src_key = if first_slash < source.len() {
|
||||
percent_decode(&source[first_slash + 1..])
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let new_metadata = if metadata_directive == "REPLACE" {
|
||||
Some(extract_metadata(req.headers()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let result = store
|
||||
.copy_object(&src_bucket, &src_key, dest_bucket, dest_key, &metadata_directive, new_metadata)
|
||||
.await?;
|
||||
|
||||
let xml = xml_response::copy_object_result_xml(&result.md5, &result.last_modified.to_rfc3339());
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Multipart handlers
|
||||
// ============================
|
||||
|
||||
async fn handle_initiate_multipart(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let metadata = extract_metadata(req.headers());
|
||||
let upload_id = store.initiate_multipart(bucket, key, metadata).await?;
|
||||
let xml = xml_response::initiate_multipart_xml(bucket, key, &upload_id);
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
async fn handle_upload_part(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
query: &HashMap<String, String>,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let upload_id = query.get("uploadId").unwrap();
|
||||
let part_number: u32 = query
|
||||
.get("partNumber")
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
if part_number < 1 || part_number > 10000 {
|
||||
return Err(S3Error::invalid_part_number().into());
|
||||
}
|
||||
|
||||
let body = req.into_body();
|
||||
let (etag, _size) = store.upload_part(upload_id, part_number, body).await?;
|
||||
|
||||
let resp = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("ETag", format!("\"{}\"", etag))
|
||||
.header("x-amz-request-id", request_id)
|
||||
.body(empty_body())
|
||||
.unwrap();
|
||||
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
async fn handle_complete_multipart(
|
||||
req: Request<Incoming>,
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
// Read request body (XML)
|
||||
let body_bytes = req.collect().await.map_err(|e| anyhow::anyhow!("Body error: {}", e))?.to_bytes();
|
||||
let body_str = String::from_utf8_lossy(&body_bytes);
|
||||
|
||||
// Parse parts from XML using regex-like approach
|
||||
let parts = parse_complete_multipart_xml(&body_str);
|
||||
|
||||
let result = store.complete_multipart(upload_id, &parts).await?;
|
||||
|
||||
let xml = xml_response::complete_multipart_xml(bucket, key, &result.etag);
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
async fn handle_abort_multipart(
|
||||
store: Arc<FileStore>,
|
||||
upload_id: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
store.abort_multipart(upload_id).await?;
|
||||
Ok(empty_response(StatusCode::NO_CONTENT, request_id))
|
||||
}
|
||||
|
||||
async fn handle_list_multipart_uploads(
|
||||
store: Arc<FileStore>,
|
||||
bucket: &str,
|
||||
request_id: &str,
|
||||
) -> Result<Response<BoxBody>> {
|
||||
let uploads = store.list_multipart_uploads(bucket).await?;
|
||||
let xml = xml_response::list_multipart_uploads_xml(bucket, &uploads);
|
||||
Ok(xml_response(StatusCode::OK, xml, request_id))
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
fn parse_query(query_string: &str) -> HashMap<String, String> {
|
||||
let mut map = HashMap::new();
|
||||
if query_string.is_empty() {
|
||||
return map;
|
||||
}
|
||||
for pair in query_string.split('&') {
|
||||
let mut parts = pair.splitn(2, '=');
|
||||
let key = parts.next().unwrap_or("");
|
||||
let value = parts.next().unwrap_or("");
|
||||
let key = percent_decode(key);
|
||||
let value = percent_decode(value);
|
||||
map.insert(key, value);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
fn percent_decode(s: &str) -> String {
|
||||
percent_encoding::percent_decode_str(s)
|
||||
.decode_utf8_lossy()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn extract_metadata(headers: &hyper::HeaderMap) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
|
||||
for (name, value) in headers {
|
||||
let name_str = name.as_str().to_lowercase();
|
||||
if let Ok(val) = value.to_str() {
|
||||
match name_str.as_str() {
|
||||
"content-type" | "cache-control" | "content-disposition"
|
||||
| "content-encoding" | "content-language" | "expires" => {
|
||||
metadata.insert(name_str, val.to_string());
|
||||
}
|
||||
_ if name_str.starts_with("x-amz-meta-") => {
|
||||
metadata.insert(name_str, val.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Default content-type
|
||||
if !metadata.contains_key("content-type") {
|
||||
metadata.insert("content-type".to_string(), "binary/octet-stream".to_string());
|
||||
}
|
||||
|
||||
metadata
|
||||
}
|
||||
|
||||
fn parse_range_header(headers: &hyper::HeaderMap) -> Option<(u64, u64)> {
|
||||
let range_val = headers.get("range")?.to_str().ok()?;
|
||||
let bytes_prefix = "bytes=";
|
||||
if !range_val.starts_with(bytes_prefix) {
|
||||
return None;
|
||||
}
|
||||
let range_spec = &range_val[bytes_prefix.len()..];
|
||||
let mut parts = range_spec.splitn(2, '-');
|
||||
let start: u64 = parts.next()?.parse().ok()?;
|
||||
let end_str = parts.next()?;
|
||||
let end: u64 = if end_str.is_empty() {
|
||||
// If no end specified, we'll handle this later based on file size
|
||||
u64::MAX
|
||||
} else {
|
||||
end_str.parse().ok()?
|
||||
};
|
||||
Some((start, end))
|
||||
}
|
||||
|
||||
fn parse_complete_multipart_xml(xml: &str) -> Vec<(u32, String)> {
|
||||
let mut parts = Vec::new();
|
||||
|
||||
// Simple XML parsing for <Part><PartNumber>N</PartNumber><ETag>...</ETag></Part>
|
||||
let mut remaining = xml;
|
||||
while let Some(part_start) = remaining.find("<Part>") {
|
||||
let after_part = &remaining[part_start + 6..];
|
||||
if let Some(part_end) = after_part.find("</Part>") {
|
||||
let part_content = &after_part[..part_end];
|
||||
|
||||
let part_number = extract_xml_value(part_content, "PartNumber")
|
||||
.and_then(|s| s.parse::<u32>().ok());
|
||||
let etag = extract_xml_value(part_content, "ETag")
|
||||
.map(|s| s.replace('"', ""));
|
||||
|
||||
if let (Some(pn), Some(et)) = (part_number, etag) {
|
||||
parts.push((pn, et));
|
||||
}
|
||||
|
||||
remaining = &after_part[part_end + 7..];
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
parts.sort_by_key(|(pn, _)| *pn);
|
||||
parts
|
||||
}
|
||||
|
||||
fn extract_xml_value<'a>(xml: &'a str, tag: &str) -> Option<String> {
|
||||
let open = format!("<{}>", tag);
|
||||
let close = format!("</{}>", tag);
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = xml.find(&close)?;
|
||||
Some(xml[start..end].to_string())
|
||||
}
|
||||
|
||||
// ============================
|
||||
// CORS
|
||||
// ============================
|
||||
|
||||
fn build_cors_preflight(config: &S3Config, request_id: &str) -> Response<BoxBody> {
|
||||
let mut builder = Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.header("x-amz-request-id", request_id);
|
||||
|
||||
if let Some(ref origins) = config.cors.allowed_origins {
|
||||
builder = builder.header("Access-Control-Allow-Origin", origins.join(", "));
|
||||
}
|
||||
if let Some(ref methods) = config.cors.allowed_methods {
|
||||
builder = builder.header("Access-Control-Allow-Methods", methods.join(", "));
|
||||
}
|
||||
if let Some(ref headers) = config.cors.allowed_headers {
|
||||
builder = builder.header("Access-Control-Allow-Headers", headers.join(", "));
|
||||
}
|
||||
if let Some(max_age) = config.cors.max_age {
|
||||
builder = builder.header("Access-Control-Max-Age", max_age.to_string());
|
||||
}
|
||||
if config.cors.allow_credentials == Some(true) {
|
||||
builder = builder.header("Access-Control-Allow-Credentials", "true");
|
||||
}
|
||||
|
||||
builder.body(empty_body()).unwrap()
|
||||
}
|
||||
|
||||
fn add_cors_headers(headers: &mut hyper::HeaderMap, config: &S3Config) {
|
||||
if let Some(ref origins) = config.cors.allowed_origins {
|
||||
headers.insert(
|
||||
"access-control-allow-origin",
|
||||
origins.join(", ").parse().unwrap(),
|
||||
);
|
||||
}
|
||||
if let Some(ref exposed) = config.cors.exposed_headers {
|
||||
headers.insert(
|
||||
"access-control-expose-headers",
|
||||
exposed.join(", ").parse().unwrap(),
|
||||
);
|
||||
}
|
||||
if config.cors.allow_credentials == Some(true) {
|
||||
headers.insert(
|
||||
"access-control-allow-credentials",
|
||||
"true".parse().unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Auth
|
||||
// ============================
|
||||
|
||||
fn check_auth(req: &Request<Incoming>, config: &S3Config) -> Result<(), S3Error> {
|
||||
let auth_header = req
|
||||
.headers()
|
||||
.get("authorization")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("");
|
||||
|
||||
if auth_header.is_empty() {
|
||||
return Err(S3Error::access_denied());
|
||||
}
|
||||
|
||||
// Extract access key from AWS v2 or v4 signature
|
||||
let access_key = if auth_header.starts_with("AWS4-HMAC-SHA256") {
|
||||
// v4: AWS4-HMAC-SHA256 Credential=KEY/date/region/s3/aws4_request, ...
|
||||
auth_header
|
||||
.split("Credential=")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('/').next())
|
||||
} else if auth_header.starts_with("AWS ") {
|
||||
// v2: AWS KEY:signature
|
||||
auth_header
|
||||
.strip_prefix("AWS ")
|
||||
.and_then(|s| s.split(':').next())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let access_key = access_key.unwrap_or("");
|
||||
|
||||
// Check against configured credentials
|
||||
for cred in &config.auth.credentials {
|
||||
if cred.access_key_id == access_key {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
Err(S3Error::access_denied())
|
||||
}
|
||||
885
rust/src/storage.rs
Normal file
885
rust/src/storage.rs
Normal file
@@ -0,0 +1,885 @@
|
||||
use anyhow::Result;
|
||||
use chrono::{DateTime, Utc};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Incoming;
|
||||
use md5::{Digest, Md5};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufWriter};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::s3_error::S3Error;
|
||||
|
||||
// ============================
|
||||
// Result types
|
||||
// ============================
|
||||
|
||||
pub struct PutResult {
|
||||
pub size: u64,
|
||||
pub md5: String,
|
||||
}
|
||||
|
||||
pub struct GetResult {
|
||||
pub key: String,
|
||||
pub size: u64,
|
||||
pub last_modified: DateTime<Utc>,
|
||||
pub md5: String,
|
||||
pub metadata: HashMap<String, String>,
|
||||
pub body: tokio::fs::File,
|
||||
pub content_length: u64,
|
||||
}
|
||||
|
||||
pub struct HeadResult {
|
||||
pub key: String,
|
||||
pub size: u64,
|
||||
pub last_modified: DateTime<Utc>,
|
||||
pub md5: String,
|
||||
pub metadata: HashMap<String, String>,
|
||||
}
|
||||
|
||||
pub struct CopyResult {
|
||||
pub size: u64,
|
||||
pub md5: String,
|
||||
pub last_modified: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct ListObjectEntry {
|
||||
pub key: String,
|
||||
pub size: u64,
|
||||
pub last_modified: DateTime<Utc>,
|
||||
pub md5: String,
|
||||
}
|
||||
|
||||
pub struct ListObjectsResult {
|
||||
pub contents: Vec<ListObjectEntry>,
|
||||
pub common_prefixes: Vec<String>,
|
||||
pub is_truncated: bool,
|
||||
pub next_continuation_token: Option<String>,
|
||||
pub prefix: String,
|
||||
pub delimiter: String,
|
||||
pub max_keys: usize,
|
||||
}
|
||||
|
||||
pub struct BucketInfo {
|
||||
pub name: String,
|
||||
pub creation_date: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct MultipartUploadInfo {
|
||||
pub upload_id: String,
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub initiated: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct CompleteMultipartResult {
|
||||
pub etag: String,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Multipart metadata (disk format, compatible with TS)
|
||||
// ============================
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct MultipartMetadata {
|
||||
upload_id: String,
|
||||
bucket: String,
|
||||
key: String,
|
||||
initiated: String,
|
||||
metadata: HashMap<String, String>,
|
||||
parts: Vec<PartMetadata>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PartMetadata {
|
||||
part_number: u32,
|
||||
etag: String,
|
||||
size: u64,
|
||||
last_modified: String,
|
||||
}
|
||||
|
||||
// ============================
|
||||
// FileStore
|
||||
// ============================
|
||||
|
||||
pub struct FileStore {
|
||||
root_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl FileStore {
|
||||
pub fn new(root_dir: PathBuf) -> Self {
|
||||
Self { root_dir }
|
||||
}
|
||||
|
||||
pub async fn initialize(&self) -> Result<()> {
|
||||
fs::create_dir_all(&self.root_dir).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reset(&self) -> Result<()> {
|
||||
if self.root_dir.exists() {
|
||||
fs::remove_dir_all(&self.root_dir).await?;
|
||||
}
|
||||
fs::create_dir_all(&self.root_dir).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Bucket operations
|
||||
// ============================
|
||||
|
||||
pub async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
let mut buckets = Vec::new();
|
||||
let mut entries = fs::read_dir(&self.root_dir).await?;
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let meta = entry.metadata().await?;
|
||||
if meta.is_dir() {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
// Skip hidden dirs like .multipart
|
||||
if name.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
let creation_date: DateTime<Utc> = meta
|
||||
.created()
|
||||
.unwrap_or(meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH))
|
||||
.into();
|
||||
buckets.push(BucketInfo {
|
||||
name,
|
||||
creation_date,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
buckets.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok(buckets)
|
||||
}
|
||||
|
||||
pub async fn bucket_exists(&self, bucket: &str) -> bool {
|
||||
self.root_dir.join(bucket).is_dir()
|
||||
}
|
||||
|
||||
pub async fn create_bucket(&self, bucket: &str) -> Result<()> {
|
||||
let bucket_path = self.root_dir.join(bucket);
|
||||
fs::create_dir_all(&bucket_path).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_bucket(&self, bucket: &str) -> Result<()> {
|
||||
let bucket_path = self.root_dir.join(bucket);
|
||||
|
||||
if !bucket_path.is_dir() {
|
||||
return Err(S3Error::no_such_bucket().into());
|
||||
}
|
||||
|
||||
// Check if bucket is empty (ignore hidden files)
|
||||
let mut entries = fs::read_dir(&bucket_path).await?;
|
||||
while let Some(_entry) = entries.next_entry().await? {
|
||||
return Err(S3Error::bucket_not_empty().into());
|
||||
}
|
||||
|
||||
fs::remove_dir_all(&bucket_path).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Object operations
|
||||
// ============================
|
||||
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
body: Incoming,
|
||||
metadata: HashMap<String, String>,
|
||||
) -> Result<PutResult> {
|
||||
if !self.bucket_exists(bucket).await {
|
||||
return Err(S3Error::no_such_bucket().into());
|
||||
}
|
||||
|
||||
let object_path = self.object_path(bucket, key);
|
||||
if let Some(parent) = object_path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
let file = fs::File::create(&object_path).await?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
let mut hasher = Md5::new();
|
||||
let mut total_size: u64 = 0;
|
||||
|
||||
// Stream body frames directly to file
|
||||
let mut body = body;
|
||||
loop {
|
||||
match body.frame().await {
|
||||
Some(Ok(frame)) => {
|
||||
if let Ok(data) = frame.into_data() {
|
||||
hasher.update(&data);
|
||||
total_size += data.len() as u64;
|
||||
writer.write_all(&data).await?;
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
return Err(anyhow::anyhow!("Body read error: {}", e));
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
writer.flush().await?;
|
||||
drop(writer);
|
||||
|
||||
let md5_hex = format!("{:x}", hasher.finalize());
|
||||
|
||||
// Write MD5 sidecar
|
||||
let md5_path = format!("{}.md5", object_path.display());
|
||||
fs::write(&md5_path, &md5_hex).await?;
|
||||
|
||||
// Write metadata sidecar
|
||||
let metadata_path = format!("{}.metadata.json", object_path.display());
|
||||
let metadata_json = serde_json::to_string_pretty(&metadata)?;
|
||||
fs::write(&metadata_path, metadata_json).await?;
|
||||
|
||||
Ok(PutResult {
|
||||
size: total_size,
|
||||
md5: md5_hex,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn put_object_bytes(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
data: &[u8],
|
||||
metadata: HashMap<String, String>,
|
||||
) -> Result<PutResult> {
|
||||
if !self.bucket_exists(bucket).await {
|
||||
return Err(S3Error::no_such_bucket().into());
|
||||
}
|
||||
|
||||
let object_path = self.object_path(bucket, key);
|
||||
if let Some(parent) = object_path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data);
|
||||
let md5_hex = format!("{:x}", hasher.finalize());
|
||||
|
||||
fs::write(&object_path, data).await?;
|
||||
|
||||
// Write MD5 sidecar
|
||||
let md5_path = format!("{}.md5", object_path.display());
|
||||
fs::write(&md5_path, &md5_hex).await?;
|
||||
|
||||
// Write metadata sidecar
|
||||
let metadata_path = format!("{}.metadata.json", object_path.display());
|
||||
let metadata_json = serde_json::to_string_pretty(&metadata)?;
|
||||
fs::write(&metadata_path, metadata_json).await?;
|
||||
|
||||
Ok(PutResult {
|
||||
size: data.len() as u64,
|
||||
md5: md5_hex,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
range: Option<(u64, u64)>,
|
||||
) -> Result<GetResult> {
|
||||
let object_path = self.object_path(bucket, key);
|
||||
|
||||
if !object_path.exists() {
|
||||
return Err(S3Error::no_such_key().into());
|
||||
}
|
||||
|
||||
let file_meta = fs::metadata(&object_path).await?;
|
||||
let size = file_meta.len();
|
||||
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
|
||||
|
||||
let md5 = self.read_md5(&object_path).await;
|
||||
let metadata = self.read_metadata(&object_path).await;
|
||||
|
||||
let mut file = fs::File::open(&object_path).await?;
|
||||
|
||||
let content_length = if let Some((start, end)) = range {
|
||||
file.seek(std::io::SeekFrom::Start(start)).await?;
|
||||
end - start + 1
|
||||
} else {
|
||||
size
|
||||
};
|
||||
|
||||
Ok(GetResult {
|
||||
key: key.to_string(),
|
||||
size,
|
||||
last_modified,
|
||||
md5,
|
||||
metadata,
|
||||
body: file,
|
||||
content_length,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn head_object(&self, bucket: &str, key: &str) -> Result<HeadResult> {
|
||||
let object_path = self.object_path(bucket, key);
|
||||
|
||||
if !object_path.exists() {
|
||||
return Err(S3Error::no_such_key().into());
|
||||
}
|
||||
|
||||
// Only stat the file, don't open it
|
||||
let file_meta = fs::metadata(&object_path).await?;
|
||||
let size = file_meta.len();
|
||||
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
|
||||
|
||||
let md5 = self.read_md5(&object_path).await;
|
||||
let metadata = self.read_metadata(&object_path).await;
|
||||
|
||||
Ok(HeadResult {
|
||||
key: key.to_string(),
|
||||
size,
|
||||
last_modified,
|
||||
md5,
|
||||
metadata,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_object(&self, bucket: &str, key: &str) -> Result<()> {
|
||||
let object_path = self.object_path(bucket, key);
|
||||
let md5_path = format!("{}.md5", object_path.display());
|
||||
let metadata_path = format!("{}.metadata.json", object_path.display());
|
||||
|
||||
// S3 doesn't error if object doesn't exist
|
||||
let _ = fs::remove_file(&object_path).await;
|
||||
let _ = fs::remove_file(&md5_path).await;
|
||||
let _ = fs::remove_file(&metadata_path).await;
|
||||
|
||||
// Clean up empty parent directories up to bucket level
|
||||
let bucket_path = self.root_dir.join(bucket);
|
||||
let mut current = object_path.parent().map(|p| p.to_path_buf());
|
||||
while let Some(dir) = current {
|
||||
if dir == bucket_path {
|
||||
break;
|
||||
}
|
||||
if fs::read_dir(&dir).await.is_ok() {
|
||||
let mut entries = fs::read_dir(&dir).await?;
|
||||
if entries.next_entry().await?.is_none() {
|
||||
let _ = fs::remove_dir(&dir).await;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
current = dir.parent().map(|p| p.to_path_buf());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn copy_object(
|
||||
&self,
|
||||
src_bucket: &str,
|
||||
src_key: &str,
|
||||
dest_bucket: &str,
|
||||
dest_key: &str,
|
||||
metadata_directive: &str,
|
||||
new_metadata: Option<HashMap<String, String>>,
|
||||
) -> Result<CopyResult> {
|
||||
let src_path = self.object_path(src_bucket, src_key);
|
||||
let dest_path = self.object_path(dest_bucket, dest_key);
|
||||
|
||||
if !src_path.exists() {
|
||||
return Err(S3Error::no_such_key().into());
|
||||
}
|
||||
|
||||
if !self.bucket_exists(dest_bucket).await {
|
||||
return Err(S3Error::no_such_bucket().into());
|
||||
}
|
||||
|
||||
if let Some(parent) = dest_path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
// Copy object file
|
||||
fs::copy(&src_path, &dest_path).await?;
|
||||
|
||||
// Handle metadata
|
||||
if metadata_directive == "COPY" {
|
||||
let src_meta_path = format!("{}.metadata.json", src_path.display());
|
||||
let dest_meta_path = format!("{}.metadata.json", dest_path.display());
|
||||
let _ = fs::copy(&src_meta_path, &dest_meta_path).await;
|
||||
} else if let Some(meta) = new_metadata {
|
||||
let dest_meta_path = format!("{}.metadata.json", dest_path.display());
|
||||
let json = serde_json::to_string_pretty(&meta)?;
|
||||
fs::write(&dest_meta_path, json).await?;
|
||||
}
|
||||
|
||||
// Copy MD5
|
||||
let src_md5_path = format!("{}.md5", src_path.display());
|
||||
let dest_md5_path = format!("{}.md5", dest_path.display());
|
||||
let _ = fs::copy(&src_md5_path, &dest_md5_path).await;
|
||||
|
||||
let file_meta = fs::metadata(&dest_path).await?;
|
||||
let md5 = self.read_md5(&dest_path).await;
|
||||
let last_modified: DateTime<Utc> = file_meta.modified()?.into();
|
||||
|
||||
Ok(CopyResult {
|
||||
size: file_meta.len(),
|
||||
md5,
|
||||
last_modified,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn list_objects(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
delimiter: &str,
|
||||
max_keys: usize,
|
||||
continuation_token: Option<&str>,
|
||||
) -> Result<ListObjectsResult> {
|
||||
let bucket_path = self.root_dir.join(bucket);
|
||||
|
||||
if !bucket_path.is_dir() {
|
||||
return Err(S3Error::no_such_bucket().into());
|
||||
}
|
||||
|
||||
// Collect all object keys recursively
|
||||
let mut keys = Vec::new();
|
||||
self.collect_keys(&bucket_path, &bucket_path, &mut keys)
|
||||
.await?;
|
||||
|
||||
// Apply prefix filter
|
||||
if !prefix.is_empty() {
|
||||
keys.retain(|k| k.starts_with(prefix));
|
||||
}
|
||||
|
||||
keys.sort();
|
||||
|
||||
// Handle continuation token
|
||||
if let Some(token) = continuation_token {
|
||||
if let Some(pos) = keys.iter().position(|k| k.as_str() > token) {
|
||||
keys = keys[pos..].to_vec();
|
||||
} else {
|
||||
keys.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Handle delimiter and pagination
|
||||
let mut common_prefixes: Vec<String> = Vec::new();
|
||||
let mut common_prefix_set = std::collections::HashSet::new();
|
||||
let mut contents: Vec<ListObjectEntry> = Vec::new();
|
||||
let mut is_truncated = false;
|
||||
|
||||
for key in &keys {
|
||||
if !delimiter.is_empty() {
|
||||
let remaining = &key[prefix.len()..];
|
||||
if let Some(delim_idx) = remaining.find(delimiter) {
|
||||
let cp = format!(
|
||||
"{}{}",
|
||||
prefix,
|
||||
&remaining[..delim_idx + delimiter.len()]
|
||||
);
|
||||
if common_prefix_set.insert(cp.clone()) {
|
||||
common_prefixes.push(cp);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if contents.len() >= max_keys {
|
||||
is_truncated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
let object_path = self.object_path(bucket, key);
|
||||
if let Ok(meta) = fs::metadata(&object_path).await {
|
||||
let md5 = self.read_md5(&object_path).await;
|
||||
let last_modified: DateTime<Utc> = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH).into();
|
||||
contents.push(ListObjectEntry {
|
||||
key: key.clone(),
|
||||
size: meta.len(),
|
||||
last_modified,
|
||||
md5,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let next_continuation_token = if is_truncated {
|
||||
contents.last().map(|e| e.key.clone())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
common_prefixes.sort();
|
||||
|
||||
Ok(ListObjectsResult {
|
||||
contents,
|
||||
common_prefixes,
|
||||
is_truncated,
|
||||
next_continuation_token,
|
||||
prefix: prefix.to_string(),
|
||||
delimiter: delimiter.to_string(),
|
||||
max_keys,
|
||||
})
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Multipart operations
|
||||
// ============================
|
||||
|
||||
fn multipart_dir(&self) -> PathBuf {
|
||||
self.root_dir.join(".multipart")
|
||||
}
|
||||
|
||||
pub async fn initiate_multipart(
|
||||
&self,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
metadata: HashMap<String, String>,
|
||||
) -> Result<String> {
|
||||
let upload_id = Uuid::new_v4().to_string().replace('-', "");
|
||||
let upload_dir = self.multipart_dir().join(&upload_id);
|
||||
fs::create_dir_all(&upload_dir).await?;
|
||||
|
||||
let meta = MultipartMetadata {
|
||||
upload_id: upload_id.clone(),
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
initiated: Utc::now().to_rfc3339(),
|
||||
metadata,
|
||||
parts: Vec::new(),
|
||||
};
|
||||
|
||||
let meta_path = upload_dir.join("metadata.json");
|
||||
let json = serde_json::to_string_pretty(&meta)?;
|
||||
fs::write(&meta_path, json).await?;
|
||||
|
||||
Ok(upload_id)
|
||||
}
|
||||
|
||||
pub async fn upload_part(
|
||||
&self,
|
||||
upload_id: &str,
|
||||
part_number: u32,
|
||||
body: Incoming,
|
||||
) -> Result<(String, u64)> {
|
||||
let upload_dir = self.multipart_dir().join(upload_id);
|
||||
if !upload_dir.is_dir() {
|
||||
return Err(S3Error::no_such_upload().into());
|
||||
}
|
||||
|
||||
let part_path = upload_dir.join(format!("part-{}", part_number));
|
||||
let file = fs::File::create(&part_path).await?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
let mut hasher = Md5::new();
|
||||
let mut size: u64 = 0;
|
||||
|
||||
let mut body = body;
|
||||
loop {
|
||||
match body.frame().await {
|
||||
Some(Ok(frame)) => {
|
||||
if let Ok(data) = frame.into_data() {
|
||||
hasher.update(&data);
|
||||
size += data.len() as u64;
|
||||
writer.write_all(&data).await?;
|
||||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
return Err(anyhow::anyhow!("Body read error: {}", e));
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
writer.flush().await?;
|
||||
drop(writer);
|
||||
|
||||
let etag = format!("{:x}", hasher.finalize());
|
||||
|
||||
// Update metadata
|
||||
self.update_multipart_metadata(upload_id, part_number, &etag, size)
|
||||
.await?;
|
||||
|
||||
Ok((etag, size))
|
||||
}
|
||||
|
||||
async fn update_multipart_metadata(
|
||||
&self,
|
||||
upload_id: &str,
|
||||
part_number: u32,
|
||||
etag: &str,
|
||||
size: u64,
|
||||
) -> Result<()> {
|
||||
let meta_path = self.multipart_dir().join(upload_id).join("metadata.json");
|
||||
let content = fs::read_to_string(&meta_path).await?;
|
||||
let mut meta: MultipartMetadata = serde_json::from_str(&content)?;
|
||||
|
||||
// Remove existing part with same number
|
||||
meta.parts.retain(|p| p.part_number != part_number);
|
||||
|
||||
meta.parts.push(PartMetadata {
|
||||
part_number,
|
||||
etag: etag.to_string(),
|
||||
size,
|
||||
last_modified: Utc::now().to_rfc3339(),
|
||||
});
|
||||
|
||||
meta.parts.sort_by_key(|p| p.part_number);
|
||||
|
||||
let json = serde_json::to_string_pretty(&meta)?;
|
||||
fs::write(&meta_path, json).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn complete_multipart(
|
||||
&self,
|
||||
upload_id: &str,
|
||||
parts: &[(u32, String)],
|
||||
) -> Result<CompleteMultipartResult> {
|
||||
let upload_dir = self.multipart_dir().join(upload_id);
|
||||
if !upload_dir.is_dir() {
|
||||
return Err(S3Error::no_such_upload().into());
|
||||
}
|
||||
|
||||
// Read metadata to get bucket/key
|
||||
let meta_path = upload_dir.join("metadata.json");
|
||||
let content = fs::read_to_string(&meta_path).await?;
|
||||
let meta: MultipartMetadata = serde_json::from_str(&content)?;
|
||||
|
||||
let object_path = self.object_path(&meta.bucket, &meta.key);
|
||||
if let Some(parent) = object_path.parent() {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
|
||||
// Concatenate parts into final object, stream each part
|
||||
let dest_file = fs::File::create(&object_path).await?;
|
||||
let mut writer = BufWriter::new(dest_file);
|
||||
let mut hasher = Md5::new();
|
||||
let mut total_size: u64 = 0;
|
||||
|
||||
for (part_number, _etag) in parts {
|
||||
let part_path = upload_dir.join(format!("part-{}", part_number));
|
||||
if !part_path.exists() {
|
||||
return Err(anyhow::anyhow!("Part {} not found", part_number));
|
||||
}
|
||||
|
||||
let mut part_file = fs::File::open(&part_path).await?;
|
||||
let mut buf = vec![0u8; 64 * 1024]; // 64KB buffer
|
||||
loop {
|
||||
let n = part_file.read(&mut buf).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
writer.write_all(&buf[..n]).await?;
|
||||
total_size += n as u64;
|
||||
}
|
||||
}
|
||||
|
||||
writer.flush().await?;
|
||||
drop(writer);
|
||||
|
||||
let etag = format!("{:x}", hasher.finalize());
|
||||
|
||||
// Write MD5 sidecar
|
||||
let md5_path = format!("{}.md5", object_path.display());
|
||||
fs::write(&md5_path, &etag).await?;
|
||||
|
||||
// Write metadata sidecar
|
||||
let metadata_path = format!("{}.metadata.json", object_path.display());
|
||||
let metadata_json = serde_json::to_string_pretty(&meta.metadata)?;
|
||||
fs::write(&metadata_path, metadata_json).await?;
|
||||
|
||||
// Clean up multipart directory
|
||||
let _ = fs::remove_dir_all(&upload_dir).await;
|
||||
|
||||
Ok(CompleteMultipartResult {
|
||||
etag,
|
||||
size: total_size,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn abort_multipart(&self, upload_id: &str) -> Result<()> {
|
||||
let upload_dir = self.multipart_dir().join(upload_id);
|
||||
if !upload_dir.is_dir() {
|
||||
return Err(S3Error::no_such_upload().into());
|
||||
}
|
||||
fs::remove_dir_all(&upload_dir).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list_multipart_uploads(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<Vec<MultipartUploadInfo>> {
|
||||
let multipart_dir = self.multipart_dir();
|
||||
if !multipart_dir.is_dir() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut uploads = Vec::new();
|
||||
let mut entries = fs::read_dir(&multipart_dir).await?;
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if !entry.metadata().await?.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let meta_path = entry.path().join("metadata.json");
|
||||
if let Ok(content) = fs::read_to_string(&meta_path).await {
|
||||
if let Ok(meta) = serde_json::from_str::<MultipartMetadata>(&content) {
|
||||
if meta.bucket == bucket {
|
||||
let initiated = DateTime::parse_from_rfc3339(&meta.initiated)
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
.unwrap_or_else(|_| Utc::now());
|
||||
|
||||
uploads.push(MultipartUploadInfo {
|
||||
upload_id: meta.upload_id,
|
||||
bucket: meta.bucket,
|
||||
key: meta.key,
|
||||
initiated,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(uploads)
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
fn object_path(&self, bucket: &str, key: &str) -> PathBuf {
|
||||
let encoded = encode_key(key);
|
||||
self.root_dir
|
||||
.join(bucket)
|
||||
.join(format!("{}._S3_object", encoded))
|
||||
}
|
||||
|
||||
async fn read_md5(&self, object_path: &Path) -> String {
|
||||
let md5_path = format!("{}.md5", object_path.display());
|
||||
match fs::read_to_string(&md5_path).await {
|
||||
Ok(s) => s.trim().to_string(),
|
||||
Err(_) => {
|
||||
// Calculate MD5 if sidecar missing
|
||||
match self.calculate_md5(object_path).await {
|
||||
Ok(hash) => {
|
||||
let _ = fs::write(&md5_path, &hash).await;
|
||||
hash
|
||||
}
|
||||
Err(_) => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn calculate_md5(&self, path: &Path) -> Result<String> {
|
||||
let mut file = fs::File::open(path).await?;
|
||||
let mut hasher = Md5::new();
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
loop {
|
||||
let n = file.read(&mut buf).await?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
async fn read_metadata(&self, object_path: &Path) -> HashMap<String, String> {
|
||||
let meta_path = format!("{}.metadata.json", object_path.display());
|
||||
match fs::read_to_string(&meta_path).await {
|
||||
Ok(s) => serde_json::from_str(&s).unwrap_or_default(),
|
||||
Err(_) => HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_keys<'a>(
|
||||
&'a self,
|
||||
bucket_path: &'a Path,
|
||||
dir: &'a Path,
|
||||
keys: &'a mut Vec<String>,
|
||||
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
let mut entries = match fs::read_dir(dir).await {
|
||||
Ok(e) => e,
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let meta = entry.metadata().await?;
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
|
||||
if meta.is_dir() {
|
||||
self.collect_keys(bucket_path, &entry.path(), keys).await?;
|
||||
} else if name.ends_with("._S3_object")
|
||||
&& !name.ends_with(".metadata.json")
|
||||
&& !name.ends_with(".md5")
|
||||
{
|
||||
let relative = entry
|
||||
.path()
|
||||
.strip_prefix(bucket_path)
|
||||
.unwrap_or(Path::new(""))
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
let key = decode_key(relative.trim_end_matches("._S3_object"));
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Key encoding (identity on Linux)
|
||||
// ============================
|
||||
|
||||
fn encode_key(key: &str) -> String {
|
||||
if cfg!(windows) {
|
||||
key.chars()
|
||||
.map(|c| match c {
|
||||
'<' | '>' | ':' | '"' | '\\' | '|' | '?' | '*' => {
|
||||
format!("&{:02x}", c as u32)
|
||||
}
|
||||
_ => c.to_string(),
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
key.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_key(encoded: &str) -> String {
|
||||
if cfg!(windows) {
|
||||
let mut result = String::new();
|
||||
let mut chars = encoded.chars();
|
||||
while let Some(c) = chars.next() {
|
||||
if c == '&' {
|
||||
let hex: String = chars.by_ref().take(2).collect();
|
||||
if let Ok(byte) = u8::from_str_radix(&hex, 16) {
|
||||
result.push(byte as char);
|
||||
} else {
|
||||
result.push('&');
|
||||
result.push_str(&hex);
|
||||
}
|
||||
} else {
|
||||
result.push(c);
|
||||
}
|
||||
}
|
||||
result
|
||||
} else {
|
||||
encoded.to_string()
|
||||
}
|
||||
}
|
||||
220
rust/src/xml_response.rs
Normal file
220
rust/src/xml_response.rs
Normal file
@@ -0,0 +1,220 @@
|
||||
use crate::storage::{BucketInfo, ListObjectsResult, MultipartUploadInfo};
|
||||
|
||||
const XML_DECL: &str = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
|
||||
const S3_NS: &str = "http://s3.amazonaws.com/doc/2006-03-01/";
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
.replace('\'', "'")
|
||||
}
|
||||
|
||||
pub fn list_buckets_xml(buckets: &[BucketInfo]) -> String {
|
||||
let mut xml = format!(
|
||||
"{}\n<ListAllMyBucketsResult xmlns=\"{}\">\
|
||||
<Owner><ID>123456789000</ID><DisplayName>S3rver</DisplayName></Owner>\
|
||||
<Buckets>",
|
||||
XML_DECL, S3_NS
|
||||
);
|
||||
|
||||
for b in buckets {
|
||||
xml.push_str(&format!(
|
||||
"<Bucket><Name>{}</Name><CreationDate>{}</CreationDate></Bucket>",
|
||||
xml_escape(&b.name),
|
||||
b.creation_date.to_rfc3339()
|
||||
));
|
||||
}
|
||||
|
||||
xml.push_str("</Buckets></ListAllMyBucketsResult>");
|
||||
xml
|
||||
}
|
||||
|
||||
pub fn list_objects_v1_xml(bucket: &str, result: &ListObjectsResult) -> String {
|
||||
let mut xml = format!(
|
||||
"{}\n<ListBucketResult xmlns=\"{}\">\
|
||||
<Name>{}</Name>\
|
||||
<Prefix>{}</Prefix>\
|
||||
<MaxKeys>{}</MaxKeys>\
|
||||
<IsTruncated>{}</IsTruncated>",
|
||||
XML_DECL,
|
||||
S3_NS,
|
||||
xml_escape(bucket),
|
||||
xml_escape(&result.prefix),
|
||||
result.max_keys,
|
||||
result.is_truncated
|
||||
);
|
||||
|
||||
if !result.delimiter.is_empty() {
|
||||
xml.push_str(&format!("<Delimiter>{}</Delimiter>", xml_escape(&result.delimiter)));
|
||||
}
|
||||
|
||||
for entry in &result.contents {
|
||||
xml.push_str(&format!(
|
||||
"<Contents>\
|
||||
<Key>{}</Key>\
|
||||
<LastModified>{}</LastModified>\
|
||||
<ETag>\"{}\"</ETag>\
|
||||
<Size>{}</Size>\
|
||||
<StorageClass>STANDARD</StorageClass>\
|
||||
</Contents>",
|
||||
xml_escape(&entry.key),
|
||||
entry.last_modified.to_rfc3339(),
|
||||
xml_escape(&entry.md5),
|
||||
entry.size
|
||||
));
|
||||
}
|
||||
|
||||
for cp in &result.common_prefixes {
|
||||
xml.push_str(&format!(
|
||||
"<CommonPrefixes><Prefix>{}</Prefix></CommonPrefixes>",
|
||||
xml_escape(cp)
|
||||
));
|
||||
}
|
||||
|
||||
xml.push_str("</ListBucketResult>");
|
||||
xml
|
||||
}
|
||||
|
||||
pub fn list_objects_v2_xml(bucket: &str, result: &ListObjectsResult) -> String {
|
||||
let mut xml = format!(
|
||||
"{}\n<ListBucketResult xmlns=\"{}\">\
|
||||
<Name>{}</Name>\
|
||||
<Prefix>{}</Prefix>\
|
||||
<MaxKeys>{}</MaxKeys>\
|
||||
<KeyCount>{}</KeyCount>\
|
||||
<IsTruncated>{}</IsTruncated>",
|
||||
XML_DECL,
|
||||
S3_NS,
|
||||
xml_escape(bucket),
|
||||
xml_escape(&result.prefix),
|
||||
result.max_keys,
|
||||
result.contents.len(),
|
||||
result.is_truncated
|
||||
);
|
||||
|
||||
if !result.delimiter.is_empty() {
|
||||
xml.push_str(&format!("<Delimiter>{}</Delimiter>", xml_escape(&result.delimiter)));
|
||||
}
|
||||
|
||||
if let Some(ref token) = result.next_continuation_token {
|
||||
xml.push_str(&format!(
|
||||
"<NextContinuationToken>{}</NextContinuationToken>",
|
||||
xml_escape(token)
|
||||
));
|
||||
}
|
||||
|
||||
for entry in &result.contents {
|
||||
xml.push_str(&format!(
|
||||
"<Contents>\
|
||||
<Key>{}</Key>\
|
||||
<LastModified>{}</LastModified>\
|
||||
<ETag>\"{}\"</ETag>\
|
||||
<Size>{}</Size>\
|
||||
<StorageClass>STANDARD</StorageClass>\
|
||||
</Contents>",
|
||||
xml_escape(&entry.key),
|
||||
entry.last_modified.to_rfc3339(),
|
||||
xml_escape(&entry.md5),
|
||||
entry.size
|
||||
));
|
||||
}
|
||||
|
||||
for cp in &result.common_prefixes {
|
||||
xml.push_str(&format!(
|
||||
"<CommonPrefixes><Prefix>{}</Prefix></CommonPrefixes>",
|
||||
xml_escape(cp)
|
||||
));
|
||||
}
|
||||
|
||||
xml.push_str("</ListBucketResult>");
|
||||
xml
|
||||
}
|
||||
|
||||
pub fn error_xml(code: &str, message: &str) -> String {
|
||||
format!(
|
||||
"{}\n<Error><Code>{}</Code><Message>{}</Message></Error>",
|
||||
XML_DECL,
|
||||
xml_escape(code),
|
||||
xml_escape(message)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn copy_object_result_xml(etag: &str, last_modified: &str) -> String {
|
||||
format!(
|
||||
"{}\n<CopyObjectResult>\
|
||||
<LastModified>{}</LastModified>\
|
||||
<ETag>\"{}\"</ETag>\
|
||||
</CopyObjectResult>",
|
||||
XML_DECL,
|
||||
xml_escape(last_modified),
|
||||
xml_escape(etag)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn initiate_multipart_xml(bucket: &str, key: &str, upload_id: &str) -> String {
|
||||
format!(
|
||||
"{}\n<InitiateMultipartUploadResult xmlns=\"{}\">\
|
||||
<Bucket>{}</Bucket>\
|
||||
<Key>{}</Key>\
|
||||
<UploadId>{}</UploadId>\
|
||||
</InitiateMultipartUploadResult>",
|
||||
XML_DECL,
|
||||
S3_NS,
|
||||
xml_escape(bucket),
|
||||
xml_escape(key),
|
||||
xml_escape(upload_id)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn complete_multipart_xml(bucket: &str, key: &str, etag: &str) -> String {
|
||||
format!(
|
||||
"{}\n<CompleteMultipartUploadResult xmlns=\"{}\">\
|
||||
<Location>/{}/{}</Location>\
|
||||
<Bucket>{}</Bucket>\
|
||||
<Key>{}</Key>\
|
||||
<ETag>\"{}\"</ETag>\
|
||||
</CompleteMultipartUploadResult>",
|
||||
XML_DECL,
|
||||
S3_NS,
|
||||
xml_escape(bucket),
|
||||
xml_escape(key),
|
||||
xml_escape(bucket),
|
||||
xml_escape(key),
|
||||
xml_escape(etag)
|
||||
)
|
||||
}
|
||||
|
||||
pub fn list_multipart_uploads_xml(bucket: &str, uploads: &[MultipartUploadInfo]) -> String {
|
||||
let mut xml = format!(
|
||||
"{}\n<ListMultipartUploadsResult xmlns=\"{}\">\
|
||||
<Bucket>{}</Bucket>\
|
||||
<KeyMarker></KeyMarker>\
|
||||
<UploadIdMarker></UploadIdMarker>\
|
||||
<MaxUploads>1000</MaxUploads>\
|
||||
<IsTruncated>false</IsTruncated>",
|
||||
XML_DECL,
|
||||
S3_NS,
|
||||
xml_escape(bucket)
|
||||
);
|
||||
|
||||
for u in uploads {
|
||||
xml.push_str(&format!(
|
||||
"<Upload>\
|
||||
<Key>{}</Key>\
|
||||
<UploadId>{}</UploadId>\
|
||||
<Initiator><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Initiator>\
|
||||
<Owner><ID>S3RVER</ID><DisplayName>S3RVER</DisplayName></Owner>\
|
||||
<StorageClass>STANDARD</StorageClass>\
|
||||
<Initiated>{}</Initiated>\
|
||||
</Upload>",
|
||||
xml_escape(&u.key),
|
||||
xml_escape(&u.upload_id),
|
||||
u.initiated.to_rfc3339()
|
||||
));
|
||||
}
|
||||
|
||||
xml.push_str("</ListMultipartUploadsResult>");
|
||||
xml
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import * as smartbucket from '@pushrocks/smartbucket';
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
|
||||
export {
|
||||
smartbucket
|
||||
};
|
||||
export { smartbucket };
|
||||
|
||||
108
test/test.aws-sdk.node.ts
Normal file
108
test/test.aws-sdk.node.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import { S3Client, CreateBucketCommand, ListBucketsCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteBucketCommand } from '@aws-sdk/client-s3';
|
||||
import { Readable } from 'stream';
|
||||
import * as smarts3 from '../ts/index.js';
|
||||
|
||||
let testSmarts3Instance: smarts3.Smarts3;
|
||||
let s3Client: S3Client;
|
||||
|
||||
// Helper to convert stream to string
|
||||
async function streamToString(stream: Readable): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
});
|
||||
}
|
||||
|
||||
tap.test('should start the S3 server and configure client', async () => {
|
||||
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
|
||||
server: {
|
||||
port: 3337,
|
||||
silent: true,
|
||||
},
|
||||
storage: {
|
||||
cleanSlate: true,
|
||||
},
|
||||
});
|
||||
|
||||
const descriptor = await testSmarts3Instance.getS3Descriptor();
|
||||
|
||||
s3Client = new S3Client({
|
||||
endpoint: `http://${descriptor.endpoint}:${descriptor.port}`,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: descriptor.accessKey,
|
||||
secretAccessKey: descriptor.accessSecret,
|
||||
},
|
||||
forcePathStyle: true,
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('should list buckets (empty)', async () => {
|
||||
const response = await s3Client.send(new ListBucketsCommand({}));
|
||||
expect(Array.isArray(response.Buckets)).toEqual(true);
|
||||
expect(response.Buckets!.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('should create a bucket', async () => {
|
||||
const response = await s3Client.send(new CreateBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
});
|
||||
|
||||
tap.test('should list buckets (showing created bucket)', async () => {
|
||||
const response = await s3Client.send(new ListBucketsCommand({}));
|
||||
expect(response.Buckets!.length).toEqual(1);
|
||||
expect(response.Buckets![0].Name).toEqual('test-bucket');
|
||||
});
|
||||
|
||||
tap.test('should upload an object', async () => {
|
||||
const response = await s3Client.send(new PutObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
Body: 'Hello from AWS SDK!',
|
||||
ContentType: 'text/plain',
|
||||
}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
expect(response.ETag).toBeTypeofString();
|
||||
});
|
||||
|
||||
tap.test('should download the object', async () => {
|
||||
const response = await s3Client.send(new GetObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
}));
|
||||
|
||||
expect(response.$metadata.httpStatusCode).toEqual(200);
|
||||
const content = await streamToString(response.Body as Readable);
|
||||
expect(content).toEqual('Hello from AWS SDK!');
|
||||
});
|
||||
|
||||
tap.test('should delete the object', async () => {
|
||||
const response = await s3Client.send(new DeleteObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
}));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should fail to get deleted object', async () => {
|
||||
await expect(
|
||||
s3Client.send(new GetObjectCommand({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-file.txt',
|
||||
}))
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
tap.test('should delete the bucket', async () => {
|
||||
const response = await s3Client.send(new DeleteBucketCommand({ Bucket: 'test-bucket' }));
|
||||
expect(response.$metadata.httpStatusCode).toEqual(204);
|
||||
});
|
||||
|
||||
tap.test('should stop the S3 server', async () => {
|
||||
await testSmarts3Instance.stop();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
25
test/test.ts
25
test/test.ts
@@ -1,28 +1,37 @@
|
||||
import { expect, tap } from '@pushrocks/tapbundle';
|
||||
import * as plugins from './plugins';
|
||||
import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
import * as smarts3 from '../ts/index';
|
||||
import * as smarts3 from '../ts/index.js';
|
||||
|
||||
let testSmarts3Instance: smarts3.Smarts3;
|
||||
|
||||
tap.test('should create a smarts3 instance and run it', async toolsArg => {
|
||||
tap.test('should create a smarts3 instance and run it', async (toolsArg) => {
|
||||
testSmarts3Instance = await smarts3.Smarts3.createAndStart({
|
||||
port: 3000,
|
||||
server: {
|
||||
port: 3333,
|
||||
},
|
||||
storage: {
|
||||
cleanSlate: true,
|
||||
},
|
||||
});
|
||||
console.log(`Let the instance run for 2 seconds`);
|
||||
await toolsArg.delayFor(2000);
|
||||
});
|
||||
|
||||
tap.test('should be able to access buckets', async () => {
|
||||
const smartbucketInstance = new plugins.smartbucket.SmartBucket(await testSmarts3Instance.getS3Descriptor());
|
||||
const smartbucketInstance = new plugins.smartbucket.SmartBucket(
|
||||
await testSmarts3Instance.getS3Descriptor(),
|
||||
);
|
||||
const bucket = await smartbucketInstance.createBucket('testbucket');
|
||||
const baseDirectory = await bucket.getBaseDirectory();
|
||||
await baseDirectory.fastStore('subdir/hello.txt', 'hi there!');
|
||||
await baseDirectory.fastPut({
|
||||
path: 'subdir/hello.txt',
|
||||
contents: 'hi there!',
|
||||
});
|
||||
});
|
||||
|
||||
tap.test('should stop the instance', async () => {
|
||||
await testSmarts3Instance.stop();
|
||||
})
|
||||
});
|
||||
|
||||
tap.start();
|
||||
|
||||
8
ts/00_commitinfo_data.ts
Normal file
8
ts/00_commitinfo_data.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
/**
|
||||
* autocreated commitinfo by @push.rocks/commitinfo
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smarts3',
|
||||
version: '5.1.1',
|
||||
description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.'
|
||||
}
|
||||
257
ts/index.ts
257
ts/index.ts
@@ -1,64 +1,249 @@
|
||||
import * as plugins from './smarts3.plugins';
|
||||
import * as paths from './paths';
|
||||
import * as plugins from './plugins.js';
|
||||
import * as paths from './paths.js';
|
||||
|
||||
export interface ISmarts3ContructorOptions {
|
||||
/**
|
||||
* Authentication configuration
|
||||
*/
|
||||
export interface IAuthConfig {
|
||||
enabled: boolean;
|
||||
credentials: Array<{
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* CORS configuration
|
||||
*/
|
||||
export interface ICorsConfig {
|
||||
enabled: boolean;
|
||||
allowedOrigins?: string[];
|
||||
allowedMethods?: string[];
|
||||
allowedHeaders?: string[];
|
||||
exposedHeaders?: string[];
|
||||
maxAge?: number;
|
||||
allowCredentials?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Logging configuration
|
||||
*/
|
||||
export interface ILoggingConfig {
|
||||
level?: 'error' | 'warn' | 'info' | 'debug';
|
||||
format?: 'text' | 'json';
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Request limits configuration
|
||||
*/
|
||||
export interface ILimitsConfig {
|
||||
maxObjectSize?: number;
|
||||
maxMetadataSize?: number;
|
||||
requestTimeout?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Multipart upload configuration
|
||||
*/
|
||||
export interface IMultipartConfig {
|
||||
expirationDays?: number;
|
||||
cleanupIntervalMinutes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Server configuration
|
||||
*/
|
||||
export interface IServerConfig {
|
||||
port?: number;
|
||||
address?: string;
|
||||
silent?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage configuration
|
||||
*/
|
||||
export interface IStorageConfig {
|
||||
directory?: string;
|
||||
cleanSlate?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete smarts3 configuration
|
||||
*/
|
||||
export interface ISmarts3Config {
|
||||
server?: IServerConfig;
|
||||
storage?: IStorageConfig;
|
||||
auth?: IAuthConfig;
|
||||
cors?: ICorsConfig;
|
||||
logging?: ILoggingConfig;
|
||||
limits?: ILimitsConfig;
|
||||
multipart?: IMultipartConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default configuration values
|
||||
*/
|
||||
const DEFAULT_CONFIG: ISmarts3Config = {
|
||||
server: {
|
||||
port: 3000,
|
||||
address: '0.0.0.0',
|
||||
silent: false,
|
||||
},
|
||||
storage: {
|
||||
directory: paths.bucketsDir,
|
||||
cleanSlate: false,
|
||||
},
|
||||
auth: {
|
||||
enabled: false,
|
||||
credentials: [
|
||||
{
|
||||
accessKeyId: 'S3RVER',
|
||||
secretAccessKey: 'S3RVER',
|
||||
},
|
||||
],
|
||||
},
|
||||
cors: {
|
||||
enabled: false,
|
||||
allowedOrigins: ['*'],
|
||||
allowedMethods: ['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'],
|
||||
allowedHeaders: ['*'],
|
||||
exposedHeaders: ['ETag', 'x-amz-request-id', 'x-amz-version-id'],
|
||||
maxAge: 86400,
|
||||
allowCredentials: false,
|
||||
},
|
||||
logging: {
|
||||
level: 'info',
|
||||
format: 'text',
|
||||
enabled: true,
|
||||
},
|
||||
limits: {
|
||||
maxObjectSize: 5 * 1024 * 1024 * 1024, // 5GB
|
||||
maxMetadataSize: 2048,
|
||||
requestTimeout: 300000, // 5 minutes
|
||||
},
|
||||
multipart: {
|
||||
expirationDays: 7,
|
||||
cleanupIntervalMinutes: 60,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Merge user config with defaults (deep merge)
|
||||
*/
|
||||
function mergeConfig(userConfig: ISmarts3Config): Required<ISmarts3Config> {
|
||||
return {
|
||||
server: {
|
||||
...DEFAULT_CONFIG.server!,
|
||||
...(userConfig.server || {}),
|
||||
},
|
||||
storage: {
|
||||
...DEFAULT_CONFIG.storage!,
|
||||
...(userConfig.storage || {}),
|
||||
},
|
||||
auth: {
|
||||
...DEFAULT_CONFIG.auth!,
|
||||
...(userConfig.auth || {}),
|
||||
},
|
||||
cors: {
|
||||
...DEFAULT_CONFIG.cors!,
|
||||
...(userConfig.cors || {}),
|
||||
},
|
||||
logging: {
|
||||
...DEFAULT_CONFIG.logging!,
|
||||
...(userConfig.logging || {}),
|
||||
},
|
||||
limits: {
|
||||
...DEFAULT_CONFIG.limits!,
|
||||
...(userConfig.limits || {}),
|
||||
},
|
||||
multipart: {
|
||||
...DEFAULT_CONFIG.multipart!,
|
||||
...(userConfig.multipart || {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* IPC command type map for RustBridge
|
||||
*/
|
||||
type TRustS3Commands = {
|
||||
start: { params: { config: Required<ISmarts3Config> }; result: {} };
|
||||
stop: { params: {}; result: {} };
|
||||
createBucket: { params: { name: string }; result: {} };
|
||||
};
|
||||
|
||||
/**
|
||||
* Main Smarts3 class - production-ready S3-compatible server
|
||||
*/
|
||||
export class Smarts3 {
|
||||
// STATIC
|
||||
public static async createAndStart(optionsArg: ConstructorParameters<typeof Smarts3>[0]) {
|
||||
const smartS3Instance = new Smarts3(optionsArg);
|
||||
public static async createAndStart(configArg: ISmarts3Config = {}) {
|
||||
const smartS3Instance = new Smarts3(configArg);
|
||||
await smartS3Instance.start();
|
||||
return smartS3Instance;
|
||||
}
|
||||
|
||||
// INSTANCE
|
||||
public options: ISmarts3ContructorOptions;
|
||||
public s3Instance: plugins.s3rver;
|
||||
public config: Required<ISmarts3Config>;
|
||||
private bridge: InstanceType<typeof plugins.RustBridge<TRustS3Commands>>;
|
||||
|
||||
constructor(optionsArg: ISmarts3ContructorOptions) {
|
||||
this.options = optionsArg;
|
||||
this.options = {
|
||||
...this.options,
|
||||
...optionsArg
|
||||
}
|
||||
constructor(configArg: ISmarts3Config = {}) {
|
||||
this.config = mergeConfig(configArg);
|
||||
this.bridge = new plugins.RustBridge<TRustS3Commands>({
|
||||
binaryName: 'rusts3',
|
||||
localPaths: [
|
||||
plugins.path.join(paths.packageDir, 'dist_rust', 'rusts3'),
|
||||
plugins.path.join(paths.packageDir, 'rust', 'target', 'release', 'rusts3'),
|
||||
plugins.path.join(paths.packageDir, 'rust', 'target', 'debug', 'rusts3'),
|
||||
],
|
||||
readyTimeoutMs: 30000,
|
||||
requestTimeoutMs: 300000,
|
||||
});
|
||||
}
|
||||
|
||||
public async start() {
|
||||
if (this.options.cleanSlate) {
|
||||
await plugins.smartfile.fs.ensureEmptyDir(paths.bucketsDir);
|
||||
} else {
|
||||
await plugins.smartfile.fs.ensureDir(paths.bucketsDir);
|
||||
const spawned = await this.bridge.spawn();
|
||||
if (!spawned) {
|
||||
throw new Error('Failed to spawn rusts3 binary. Make sure it is compiled (pnpm build).');
|
||||
}
|
||||
this.s3Instance = new plugins.s3rver({
|
||||
port: this.options.port || 3000,
|
||||
address: '0.0.0.0',
|
||||
silent: false,
|
||||
directory: paths.bucketsDir
|
||||
})
|
||||
await this.s3Instance.run();
|
||||
await this.bridge.sendCommand('start', { config: this.config });
|
||||
|
||||
if (!this.config.server.silent) {
|
||||
console.log('s3 server is running');
|
||||
}
|
||||
|
||||
public async getS3Descriptor(): Promise<plugins.smartbucket.ISmartBucketConfig> {
|
||||
return {
|
||||
accessKey: 'S3RVER',
|
||||
accessSecret: 'S3RVER',
|
||||
endpoint: 'localhost',
|
||||
port: this.options.port,
|
||||
useSsl: false,
|
||||
}
|
||||
|
||||
public async getS3Descriptor(
|
||||
optionsArg?: Partial<plugins.tsclass.storage.IS3Descriptor>,
|
||||
): Promise<plugins.tsclass.storage.IS3Descriptor> {
|
||||
const cred = this.config.auth.credentials[0] || {
|
||||
accessKeyId: 'S3RVER',
|
||||
secretAccessKey: 'S3RVER',
|
||||
};
|
||||
|
||||
const descriptor: plugins.tsclass.storage.IS3Descriptor = {
|
||||
endpoint: this.config.server.address === '0.0.0.0' ? 'localhost' : this.config.server.address!,
|
||||
port: this.config.server.port!,
|
||||
useSsl: false,
|
||||
accessKey: cred.accessKeyId,
|
||||
accessSecret: cred.secretAccessKey,
|
||||
bucketName: '',
|
||||
};
|
||||
|
||||
return {
|
||||
...descriptor,
|
||||
...(optionsArg ? optionsArg : {}),
|
||||
};
|
||||
}
|
||||
|
||||
public async createBucket(bucketNameArg: string) {
|
||||
const smartbucketInstance = new plugins.smartbucket.SmartBucket(await this.getS3Descriptor());
|
||||
const bucket = await smartbucketInstance.createBucket(bucketNameArg);
|
||||
return bucket;
|
||||
await this.bridge.sendCommand('createBucket', { name: bucketNameArg });
|
||||
return { name: bucketNameArg };
|
||||
}
|
||||
|
||||
public async stop() {
|
||||
await this.s3Instance.close();
|
||||
await this.bridge.sendCommand('stop', {});
|
||||
this.bridge.kill();
|
||||
}
|
||||
}
|
||||
11
ts/paths.ts
11
ts/paths.ts
@@ -1,6 +1,11 @@
|
||||
import * as plugins from './smarts3.plugins';
|
||||
import * as plugins from './plugins.js';
|
||||
|
||||
export const packageDir = plugins.path.join(__dirname, '../');
|
||||
export const packageDir = plugins.path.join(
|
||||
plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
|
||||
'../',
|
||||
);
|
||||
export const nogitDir = plugins.path.join(packageDir, './.nogit');
|
||||
|
||||
export const bucketsDir = plugins.path.join(nogitDir, './bucketsDir');
|
||||
export const cwd = process.cwd();
|
||||
|
||||
export const bucketsDir = plugins.path.join(cwd, '.nogit/bucketsDir');
|
||||
|
||||
15
ts/plugins.ts
Normal file
15
ts/plugins.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
// node native
|
||||
import * as path from 'path';
|
||||
|
||||
export { path };
|
||||
|
||||
// @push.rocks scope
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import { RustBridge } from '@push.rocks/smartrust';
|
||||
|
||||
export { smartpath, RustBridge };
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
export { tsclass };
|
||||
@@ -1,22 +0,0 @@
|
||||
// node native
|
||||
import * as path from 'path';
|
||||
|
||||
export {
|
||||
path
|
||||
}
|
||||
|
||||
// pushrocks scope
|
||||
import * as smartbucket from '@pushrocks/smartbucket';
|
||||
import * as smartfile from '@pushrocks/smartfile';
|
||||
|
||||
export {
|
||||
smartbucket,
|
||||
smartfile,
|
||||
}
|
||||
|
||||
// thirdparty scope
|
||||
import s3rver from 's3rver';
|
||||
|
||||
export {
|
||||
s3rver
|
||||
}
|
||||
12
tsconfig.json
Normal file
12
tsconfig.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"esModuleInterop": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {}
|
||||
},
|
||||
"exclude": ["dist_*/**/*.d.ts"]
|
||||
}
|
||||
17
tslint.json
17
tslint.json
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"extends": ["tslint:latest", "tslint-config-prettier"],
|
||||
"rules": {
|
||||
"semicolon": [true, "always"],
|
||||
"no-console": false,
|
||||
"ordered-imports": false,
|
||||
"object-literal-sort-keys": false,
|
||||
"member-ordering": {
|
||||
"options":{
|
||||
"order": [
|
||||
"static-method"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"defaultSeverity": "warning"
|
||||
}
|
||||
Reference in New Issue
Block a user