fix(filesystem): Migrate filesystem implementation to @push.rocks/smartfs and add Web Streams handling
This commit is contained in:
10
changelog.md
10
changelog.md
@@ -1,5 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## 2025-11-23 - 3.0.3 - fix(filesystem)
|
||||
Migrate filesystem implementation to @push.rocks/smartfs and add Web Streams handling
|
||||
|
||||
- Replace dependency @push.rocks/smartfile with @push.rocks/smartfs and update README references
|
||||
- plugins: instantiate SmartFs with SmartFsProviderNode and export smartfs (remove direct fs export)
|
||||
- Refactor FilesystemStore to use smartfs directory/file APIs for initialize, reset, list, read, write, copy and delete
|
||||
- Implement Web Stream ↔ Node.js stream conversion for uploads/downloads (Readable.fromWeb and writer.write with Uint8Array)
|
||||
- Persist and read metadata (.metadata.json) and cached MD5 (.md5) via smartfs APIs
|
||||
- Update readme.hints and documentation to note successful migration and next steps
|
||||
|
||||
## 2025-11-21 - 3.0.2 - fix(smarts3)
|
||||
Prepare patch release 3.0.2 — no code changes detected
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
],
|
||||
"dependencies": {
|
||||
"@push.rocks/smartbucket": "^4.3.0",
|
||||
"@push.rocks/smartfile": "^11.2.7",
|
||||
"@push.rocks/smartfs": "^1.1.0",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartxml": "^2.0.0",
|
||||
"@tsclass/tsclass": "^9.3.0"
|
||||
|
||||
13
pnpm-lock.yaml
generated
13
pnpm-lock.yaml
generated
@@ -11,9 +11,9 @@ importers:
|
||||
'@push.rocks/smartbucket':
|
||||
specifier: ^4.3.0
|
||||
version: 4.3.0
|
||||
'@push.rocks/smartfile':
|
||||
specifier: ^11.2.7
|
||||
version: 11.2.7
|
||||
'@push.rocks/smartfs':
|
||||
specifier: ^1.1.0
|
||||
version: 1.1.0
|
||||
'@push.rocks/smartpath':
|
||||
specifier: ^6.0.0
|
||||
version: 6.0.0
|
||||
@@ -644,6 +644,9 @@ packages:
|
||||
'@push.rocks/smartfile@11.2.7':
|
||||
resolution: {integrity: sha512-8Yp7/sAgPpWJBHohV92ogHWKzRomI5MEbSG6b5W2n18tqwfAmjMed0rQvsvGrSBlnEWCKgoOrYIIZbLO61+J0Q==}
|
||||
|
||||
'@push.rocks/smartfs@1.1.0':
|
||||
resolution: {integrity: sha512-fg8JIjFUPPX5laRoBpTaGwhMfZ3Y8mFT4fUaW54Y4J/BfOBa/y0+rIFgvgvqcOZgkQlyZU+FIfL8Z6zezqxyTg==}
|
||||
|
||||
'@push.rocks/smartguard@3.1.0':
|
||||
resolution: {integrity: sha512-J23q84f1O+TwFGmd4lrO9XLHUh2DaLXo9PN/9VmTWYzTkQDv5JehmifXVI0esophXcCIfbdIu6hbt7/aHlDF4A==}
|
||||
|
||||
@@ -4920,6 +4923,10 @@ snapshots:
|
||||
glob: 11.1.0
|
||||
js-yaml: 4.1.1
|
||||
|
||||
'@push.rocks/smartfs@1.1.0':
|
||||
dependencies:
|
||||
'@push.rocks/smartpath': 6.0.0
|
||||
|
||||
'@push.rocks/smartguard@3.1.0':
|
||||
dependencies:
|
||||
'@push.rocks/smartpromise': 4.2.3
|
||||
|
||||
438
production-readiness.md
Normal file
438
production-readiness.md
Normal file
@@ -0,0 +1,438 @@
|
||||
# Production-Readiness Plan for smarts3
|
||||
|
||||
**Goal:** Make smarts3 production-ready as a MinIO alternative for use cases where:
|
||||
- Running MinIO is out of scope
|
||||
- You have a program written for S3 and want to use the local filesystem
|
||||
- You need a lightweight, zero-dependency S3-compatible server
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Current State Analysis
|
||||
|
||||
### ✅ What's Working
|
||||
|
||||
- **Native S3 server** with zero framework dependencies
|
||||
- **Core S3 operations:** PUT, GET, HEAD, DELETE (objects & buckets)
|
||||
- **List buckets and objects** (V1 and V2 API)
|
||||
- **Object copy** with metadata handling
|
||||
- **Range requests** for partial downloads
|
||||
- **MD5 checksums** and ETag support
|
||||
- **Custom metadata** (x-amz-meta-*)
|
||||
- **Filesystem-backed storage** with Windows compatibility
|
||||
- **S3-compatible XML error responses**
|
||||
- **Middleware system** and routing
|
||||
- **AWS SDK v3 compatibility** (tested)
|
||||
|
||||
### ❌ Production Gaps Identified
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Critical Features (Required for Production)
|
||||
|
||||
### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY**
|
||||
|
||||
**Why:** Essential for uploading files >5MB efficiently. Without this, smarts3 can't handle real-world production workloads.
|
||||
|
||||
**Implementation Required:**
|
||||
- `POST /:bucket/:key?uploads` - CreateMultipartUpload
|
||||
- `PUT /:bucket/:key?partNumber=X&uploadId=Y` - UploadPart
|
||||
- `POST /:bucket/:key?uploadId=X` - CompleteMultipartUpload
|
||||
- `DELETE /:bucket/:key?uploadId=X` - AbortMultipartUpload
|
||||
- `GET /:bucket/:key?uploadId=X` - ListParts
|
||||
- Multipart state management (temp storage for parts)
|
||||
- Part ETag tracking and validation
|
||||
- Automatic cleanup of abandoned uploads
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/multipart.controller.ts` (new)
|
||||
- `ts/classes/filesystem-store.ts` (add multipart methods)
|
||||
- `ts/classes/smarts3-server.ts` (add multipart routes)
|
||||
|
||||
---
|
||||
|
||||
### 2. Configurable Authentication 🔐
|
||||
|
||||
**Why:** Currently hardcoded credentials ('S3RVER'/'S3RVER'). Production needs custom credentials.
|
||||
|
||||
**Implementation Required:**
|
||||
- Support custom access keys and secrets via configuration
|
||||
- Implement AWS Signature V4 verification
|
||||
- Support multiple credential pairs (IAM-like users)
|
||||
- Optional: Disable authentication for local dev use
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface IAuthConfig {
|
||||
enabled: boolean;
|
||||
credentials: Array<{
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}>;
|
||||
signatureVersion: 'v4' | 'none';
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/auth-middleware.ts` (new)
|
||||
- `ts/classes/signature-validator.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate auth middleware)
|
||||
- `ts/index.ts` (add auth config options)
|
||||
|
||||
---
|
||||
|
||||
### 3. CORS Support 🌐
|
||||
|
||||
**Why:** Required for browser-based uploads and modern web apps.
|
||||
|
||||
**Implementation Required:**
|
||||
- Add CORS middleware
|
||||
- Support preflight OPTIONS requests
|
||||
- Configurable CORS origins, methods, headers
|
||||
- Per-bucket CORS configuration (optional)
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface ICorsConfig {
|
||||
enabled: boolean;
|
||||
allowedOrigins: string[]; // ['*'] or ['https://example.com']
|
||||
allowedMethods: string[]; // ['GET', 'POST', 'PUT', 'DELETE']
|
||||
allowedHeaders: string[]; // ['*'] or specific headers
|
||||
exposedHeaders: string[]; // ['ETag', 'x-amz-*']
|
||||
maxAge: number; // 3600 (seconds)
|
||||
allowCredentials: boolean;
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/cors-middleware.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate CORS middleware)
|
||||
- `ts/index.ts` (add CORS config options)
|
||||
|
||||
---
|
||||
|
||||
### 4. SSL/TLS Support 🔒
|
||||
|
||||
**Why:** Production systems require encrypted connections.
|
||||
|
||||
**Implementation Required:**
|
||||
- HTTPS server option with cert/key configuration
|
||||
- Auto-redirect HTTP to HTTPS (optional)
|
||||
- Support for self-signed certs in dev mode
|
||||
|
||||
**Configuration Example:**
|
||||
```typescript
|
||||
interface ISslConfig {
|
||||
enabled: boolean;
|
||||
cert: string; // Path to certificate file or cert content
|
||||
key: string; // Path to key file or key content
|
||||
ca?: string; // Optional CA cert
|
||||
redirectHttp?: boolean; // Redirect HTTP to HTTPS
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/smarts3-server.ts` (add HTTPS server creation)
|
||||
- `ts/index.ts` (add SSL config options)
|
||||
|
||||
---
|
||||
|
||||
### 5. Production Configuration System ⚙️
|
||||
|
||||
**Why:** Production needs flexible configuration, not just constructor options.
|
||||
|
||||
**Implementation Required:**
|
||||
- Support configuration file (JSON/YAML)
|
||||
- Environment variable support
|
||||
- Configuration validation
|
||||
- Sensible production defaults
|
||||
- Example configurations for common use cases
|
||||
|
||||
**Configuration File Example (`smarts3.config.json`):**
|
||||
```json
|
||||
{
|
||||
"server": {
|
||||
"port": 3000,
|
||||
"address": "0.0.0.0",
|
||||
"ssl": {
|
||||
"enabled": true,
|
||||
"cert": "./certs/server.crt",
|
||||
"key": "./certs/server.key"
|
||||
}
|
||||
},
|
||||
"storage": {
|
||||
"directory": "./s3-data",
|
||||
"cleanSlate": false
|
||||
},
|
||||
"auth": {
|
||||
"enabled": true,
|
||||
"credentials": [
|
||||
{
|
||||
"accessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cors": {
|
||||
"enabled": true,
|
||||
"allowedOrigins": ["*"],
|
||||
"allowedMethods": ["GET", "POST", "PUT", "DELETE", "HEAD"],
|
||||
"allowedHeaders": ["*"]
|
||||
},
|
||||
"limits": {
|
||||
"maxObjectSize": 5368709120,
|
||||
"maxMetadataSize": 2048,
|
||||
"requestTimeout": 300000
|
||||
},
|
||||
"logging": {
|
||||
"level": "info",
|
||||
"format": "json",
|
||||
"accessLog": {
|
||||
"enabled": true,
|
||||
"path": "./logs/access.log"
|
||||
},
|
||||
"errorLog": {
|
||||
"enabled": true,
|
||||
"path": "./logs/error.log"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/config-loader.ts` (new)
|
||||
- `ts/classes/config-validator.ts` (new)
|
||||
- `ts/index.ts` (use config loader)
|
||||
- Create example config files in root
|
||||
|
||||
---
|
||||
|
||||
### 6. Production Logging 📝
|
||||
|
||||
**Why:** Console logs aren't suitable for production monitoring.
|
||||
|
||||
**Implementation Required:**
|
||||
- Structured logging (JSON format option)
|
||||
- Log levels (ERROR, WARN, INFO, DEBUG)
|
||||
- File rotation support
|
||||
- Access logs (S3 standard format)
|
||||
- Integration with logging library
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?)
|
||||
- `ts/classes/access-logger-middleware.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (replace console.log with logger)
|
||||
- All controller files (use structured logging)
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Important Features (Should Have)
|
||||
|
||||
### 7. Health Check & Metrics 💊
|
||||
|
||||
**Implementation Required:**
|
||||
- `GET /_health` endpoint (non-S3, for monitoring)
|
||||
- `GET /_metrics` endpoint (Prometheus format?)
|
||||
- Server stats (requests/sec, storage used, uptime)
|
||||
- Readiness/liveness probes for Kubernetes
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/health.controller.ts` (new)
|
||||
- `ts/classes/metrics-collector.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (add health routes)
|
||||
|
||||
---
|
||||
|
||||
### 8. Batch Operations 📦
|
||||
|
||||
**Implementation Required:**
|
||||
- `POST /:bucket?delete` - DeleteObjects (delete multiple objects in one request)
|
||||
- Essential for efficient cleanup operations
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/object.controller.ts` (add deleteObjects method)
|
||||
|
||||
---
|
||||
|
||||
### 9. Request Size Limits & Validation 🛡️
|
||||
|
||||
**Implementation Required:**
|
||||
- Max object size configuration
|
||||
- Max metadata size limits
|
||||
- Request timeout configuration
|
||||
- Body size limits
|
||||
- Bucket name validation (S3 rules)
|
||||
- Key name validation
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/validation-middleware.ts` (new)
|
||||
- `ts/utils/validators.ts` (new)
|
||||
- `ts/classes/smarts3-server.ts` (integrate validation middleware)
|
||||
|
||||
---
|
||||
|
||||
### 10. Conditional Requests 🔄
|
||||
|
||||
**Implementation Required:**
|
||||
- If-Match / If-None-Match (ETag validation)
|
||||
- If-Modified-Since / If-Unmodified-Since
|
||||
- Required for caching and conflict prevention
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/controllers/object.controller.ts` (add conditional logic to GET/HEAD)
|
||||
|
||||
---
|
||||
|
||||
### 11. Graceful Shutdown 👋
|
||||
|
||||
**Implementation Required:**
|
||||
- Drain existing connections
|
||||
- Reject new connections
|
||||
- Clean multipart cleanup on shutdown
|
||||
- SIGTERM/SIGINT handling
|
||||
|
||||
**Files to Create/Modify:**
|
||||
- `ts/classes/smarts3-server.ts` (add graceful shutdown logic)
|
||||
- `ts/index.ts` (add signal handlers)
|
||||
|
||||
---
|
||||
|
||||
## 💡 Nice-to-Have Features
|
||||
|
||||
### 12. Advanced Features
|
||||
|
||||
- Bucket versioning support
|
||||
- Object tagging
|
||||
- Lifecycle policies (auto-delete old objects)
|
||||
- Storage class simulation (STANDARD, GLACIER, etc.)
|
||||
- Server-side encryption simulation
|
||||
- Presigned URL support (for time-limited access)
|
||||
|
||||
### 13. Performance Optimizations
|
||||
|
||||
- Stream optimization for large files
|
||||
- Optional in-memory caching for small objects
|
||||
- Parallel upload/download support
|
||||
- Compression support (gzip)
|
||||
|
||||
### 14. Developer Experience
|
||||
|
||||
- Docker image for easy deployment
|
||||
- Docker Compose examples
|
||||
- Kubernetes manifests
|
||||
- CLI for server management
|
||||
- Admin API for bucket management
|
||||
|
||||
---
|
||||
|
||||
## 📐 Implementation Phases
|
||||
|
||||
### Phase 1: Critical Production Features (Priority 1)
|
||||
|
||||
**Estimated Effort:** 2-3 weeks
|
||||
|
||||
1. ✅ Multipart uploads (biggest technical lift)
|
||||
2. ✅ Configurable authentication
|
||||
3. ✅ CORS middleware
|
||||
4. ✅ Production configuration system
|
||||
5. ✅ Production logging
|
||||
|
||||
**Outcome:** smarts3 can handle real production workloads
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Reliability & Operations (Priority 2)
|
||||
|
||||
**Estimated Effort:** 1-2 weeks
|
||||
|
||||
6. ✅ SSL/TLS support
|
||||
7. ✅ Health checks & metrics
|
||||
8. ✅ Request validation & limits
|
||||
9. ✅ Graceful shutdown
|
||||
10. ✅ Batch operations
|
||||
|
||||
**Outcome:** smarts3 is operationally mature
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: S3 Compatibility (Priority 3)
|
||||
|
||||
**Estimated Effort:** 1-2 weeks
|
||||
|
||||
11. ✅ Conditional requests
|
||||
12. ✅ Additional S3 features as needed
|
||||
13. ✅ Comprehensive test suite
|
||||
14. ✅ Documentation updates
|
||||
|
||||
**Outcome:** smarts3 has broad S3 API compatibility
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Polish (Priority 4)
|
||||
|
||||
**Estimated Effort:** As needed
|
||||
|
||||
15. ✅ Docker packaging
|
||||
16. ✅ Performance optimization
|
||||
17. ✅ Advanced features based on user feedback
|
||||
|
||||
**Outcome:** smarts3 is a complete MinIO alternative
|
||||
|
||||
---
|
||||
|
||||
## 🤔 Open Questions
|
||||
|
||||
1. **Authentication:** Do you want full AWS Signature V4 validation, or simpler static credential checking?
|
||||
2. **Configuration:** Prefer JSON, YAML, or .env file format?
|
||||
3. **Logging:** Do you have a preferred logging library, or shall I use @push.rocks/smartlog?
|
||||
4. **Scope:** Should we tackle all of Phase 1, or start with a subset (e.g., just multipart + auth)?
|
||||
5. **Testing:** Should we add comprehensive tests as we go, or batch them at the end?
|
||||
6. **Breaking changes:** Can I modify the constructor options interface, or must it remain backward compatible?
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Target Use Cases
|
||||
|
||||
**With this plan implemented, smarts3 will be a solid MinIO alternative for:**
|
||||
|
||||
✅ **Local S3 development** - Fast, simple, no Docker required
|
||||
✅ **Testing S3 integrations** - Reliable, repeatable tests
|
||||
✅ **Microservices using S3 API** with filesystem backend
|
||||
✅ **CI/CD pipelines** - Lightweight S3 for testing
|
||||
✅ **Small-to-medium production deployments** where MinIO is overkill
|
||||
✅ **Edge computing** - S3 API for local file storage
|
||||
✅ **Embedded systems** - Minimal dependencies, small footprint
|
||||
|
||||
---
|
||||
|
||||
## 📊 Current vs. Production Comparison
|
||||
|
||||
| Feature | Current | After Phase 1 | After Phase 2 | Production Ready |
|
||||
|---------|---------|---------------|---------------|------------------|
|
||||
| Basic S3 ops | ✅ | ✅ | ✅ | ✅ |
|
||||
| Multipart upload | ❌ | ✅ | ✅ | ✅ |
|
||||
| Authentication | ⚠️ (hardcoded) | ✅ | ✅ | ✅ |
|
||||
| CORS | ❌ | ✅ | ✅ | ✅ |
|
||||
| SSL/TLS | ❌ | ❌ | ✅ | ✅ |
|
||||
| Config files | ❌ | ✅ | ✅ | ✅ |
|
||||
| Production logging | ⚠️ (console) | ✅ | ✅ | ✅ |
|
||||
| Health checks | ❌ | ❌ | ✅ | ✅ |
|
||||
| Request limits | ❌ | ❌ | ✅ | ✅ |
|
||||
| Graceful shutdown | ❌ | ❌ | ✅ | ✅ |
|
||||
| Conditional requests | ❌ | ❌ | ❌ | ✅ |
|
||||
| Batch operations | ❌ | ❌ | ✅ | ✅ |
|
||||
|
||||
---
|
||||
|
||||
## 📝 Notes
|
||||
|
||||
- All features should maintain backward compatibility where possible
|
||||
- Each feature should include comprehensive tests
|
||||
- Documentation (readme.md) should be updated as features are added
|
||||
- Consider adding a migration guide for users upgrading from testing to production use
|
||||
- Performance benchmarks should be established and maintained
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** 2025-11-23
|
||||
**Status:** Planning Phase
|
||||
**Next Step:** Get approval and prioritize implementation order
|
||||
@@ -1 +1,74 @@
|
||||
# Project Hints for smarts3
|
||||
|
||||
## Current State (v3.0.0)
|
||||
|
||||
- Native custom S3 server implementation (Smarts3Server)
|
||||
- No longer uses legacy s3rver backend (removed in v3.0.0)
|
||||
- Core S3 operations working: PUT, GET, HEAD, DELETE for objects and buckets
|
||||
- Multipart upload NOT yet implemented (critical gap for production)
|
||||
- Authentication is hardcoded ('S3RVER'/'S3RVER') - not production-ready
|
||||
- No CORS support yet
|
||||
- No SSL/TLS support yet
|
||||
|
||||
## Production Readiness
|
||||
|
||||
See `production-readiness.md` for the complete gap analysis and implementation plan.
|
||||
|
||||
**Key Missing Features for Production:**
|
||||
1. Multipart upload support (HIGHEST PRIORITY)
|
||||
2. Configurable authentication
|
||||
3. CORS middleware
|
||||
4. SSL/TLS support
|
||||
5. Production configuration system
|
||||
6. Production logging
|
||||
|
||||
## Architecture Notes
|
||||
|
||||
### File Structure
|
||||
- `ts/classes/smarts3-server.ts` - Main server class
|
||||
- `ts/classes/filesystem-store.ts` - Storage layer (filesystem-backed)
|
||||
- `ts/classes/router.ts` - URL routing with pattern matching
|
||||
- `ts/classes/middleware-stack.ts` - Middleware execution
|
||||
- `ts/classes/context.ts` - Request/response context
|
||||
- `ts/classes/s3-error.ts` - S3-compatible error handling
|
||||
- `ts/controllers/` - Service, bucket, and object controllers
|
||||
- `ts/index.ts` - Main export (Smarts3 class)
|
||||
|
||||
### Storage Layout
|
||||
- Objects stored as: `{bucket}/{encodedKey}._S3_object`
|
||||
- Metadata stored as: `{bucket}/{encodedKey}._S3_object.metadata.json`
|
||||
- MD5 stored as: `{bucket}/{encodedKey}._S3_object.md5`
|
||||
- Keys are encoded for Windows compatibility (hex encoding for invalid chars)
|
||||
|
||||
### Current Limitations
|
||||
- Max file size limited by available memory (no streaming multipart)
|
||||
- Single server instance only (no clustering)
|
||||
- No versioning support
|
||||
- No access control beyond basic auth
|
||||
|
||||
## Testing
|
||||
|
||||
- Main test: `test/test.aws-sdk.node.ts` - Tests AWS SDK v3 compatibility
|
||||
- Run with: `pnpm test`
|
||||
- Tests run with cleanSlate mode enabled
|
||||
|
||||
## Dependencies
|
||||
|
||||
- `@push.rocks/smartbucket` - S3 abstraction layer
|
||||
- `@push.rocks/smartfs` - Modern filesystem operations with Web Streams API (replaced smartfile)
|
||||
- `@push.rocks/smartxml` - XML generation/parsing
|
||||
- `@push.rocks/smartpath` - Path utilities
|
||||
- `@tsclass/tsclass` - TypeScript utilities
|
||||
|
||||
## Migration Notes (2025-11-23)
|
||||
|
||||
Successfully migrated from `@push.rocks/smartfile` + native `fs` to `@push.rocks/smartfs`:
|
||||
- All file/directory operations now use smartfs fluent API
|
||||
- Web Streams → Node.js Streams conversion for HTTP compatibility
|
||||
- All tests passing ✅
|
||||
- Build successful ✅
|
||||
|
||||
## Next Steps
|
||||
|
||||
Waiting for approval to proceed with production-readiness implementation.
|
||||
Priority 1 is implementing multipart uploads.
|
||||
|
||||
@@ -413,7 +413,7 @@ interface ISmarts3ContructorOptions {
|
||||
## 🔗 Related Packages
|
||||
|
||||
- [`@push.rocks/smartbucket`](https://www.npmjs.com/package/@push.rocks/smartbucket) - Powerful S3 abstraction layer
|
||||
- [`@push.rocks/smartfile`](https://www.npmjs.com/package/@push.rocks/smartfile) - Advanced file system operations
|
||||
- [`@push.rocks/smartfs`](https://www.npmjs.com/package/@push.rocks/smartfs) - Modern filesystem with Web Streams support
|
||||
- [`@tsclass/tsclass`](https://www.npmjs.com/package/@tsclass/tsclass) - TypeScript class helpers
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smarts3',
|
||||
version: '3.0.2',
|
||||
version: '3.0.3',
|
||||
description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.'
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as plugins from '../plugins.js';
|
||||
import { S3Error } from './s3-error.js';
|
||||
import type { Readable } from 'stream';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export interface IS3Bucket {
|
||||
name: string;
|
||||
@@ -39,7 +39,7 @@ export interface IRangeOptions {
|
||||
}
|
||||
|
||||
/**
|
||||
* Filesystem-backed storage for S3 objects
|
||||
* Filesystem-backed storage for S3 objects using smartfs
|
||||
*/
|
||||
export class FilesystemStore {
|
||||
constructor(private rootDir: string) {}
|
||||
@@ -48,14 +48,19 @@ export class FilesystemStore {
|
||||
* Initialize store (ensure root directory exists)
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
await plugins.fs.promises.mkdir(this.rootDir, { recursive: true });
|
||||
await plugins.smartfs.directory(this.rootDir).recursive().create();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset store (delete all buckets)
|
||||
*/
|
||||
public async reset(): Promise<void> {
|
||||
await plugins.smartfile.fs.ensureEmptyDir(this.rootDir);
|
||||
// Delete directory and recreate it
|
||||
const exists = await plugins.smartfs.directory(this.rootDir).exists();
|
||||
if (exists) {
|
||||
await plugins.smartfs.directory(this.rootDir).recursive().delete();
|
||||
}
|
||||
await plugins.smartfs.directory(this.rootDir).recursive().create();
|
||||
}
|
||||
|
||||
// ============================
|
||||
@@ -66,18 +71,17 @@ export class FilesystemStore {
|
||||
* List all buckets
|
||||
*/
|
||||
public async listBuckets(): Promise<IS3Bucket[]> {
|
||||
const dirs = await plugins.smartfile.fs.listFolders(this.rootDir);
|
||||
const entries = await plugins.smartfs.directory(this.rootDir).includeStats().list();
|
||||
const buckets: IS3Bucket[] = [];
|
||||
|
||||
for (const dir of dirs) {
|
||||
const bucketPath = plugins.path.join(this.rootDir, dir);
|
||||
const stats = await plugins.smartfile.fs.stat(bucketPath);
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory && entry.stats) {
|
||||
buckets.push({
|
||||
name: dir,
|
||||
creationDate: stats.birthtime,
|
||||
name: entry.name,
|
||||
creationDate: entry.stats.birthtime,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return buckets.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
@@ -87,7 +91,7 @@ export class FilesystemStore {
|
||||
*/
|
||||
public async bucketExists(bucket: string): Promise<boolean> {
|
||||
const bucketPath = this.getBucketPath(bucket);
|
||||
return plugins.smartfile.fs.isDirectory(bucketPath);
|
||||
return plugins.smartfs.directory(bucketPath).exists();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -95,7 +99,7 @@ export class FilesystemStore {
|
||||
*/
|
||||
public async createBucket(bucket: string): Promise<void> {
|
||||
const bucketPath = this.getBucketPath(bucket);
|
||||
await plugins.fs.promises.mkdir(bucketPath, { recursive: true });
|
||||
await plugins.smartfs.directory(bucketPath).recursive().create();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -110,12 +114,12 @@ export class FilesystemStore {
|
||||
}
|
||||
|
||||
// Check if bucket is empty
|
||||
const files = await plugins.smartfile.fs.listFileTree(bucketPath, '**/*');
|
||||
const files = await plugins.smartfs.directory(bucketPath).recursive().list();
|
||||
if (files.length > 0) {
|
||||
throw new S3Error('BucketNotEmpty', 'The bucket you tried to delete is not empty');
|
||||
}
|
||||
|
||||
await plugins.smartfile.fs.remove(bucketPath);
|
||||
await plugins.smartfs.directory(bucketPath).recursive().delete();
|
||||
}
|
||||
|
||||
// ============================
|
||||
@@ -142,13 +146,16 @@ export class FilesystemStore {
|
||||
continuationToken,
|
||||
} = options;
|
||||
|
||||
// List all object files
|
||||
const objectPattern = '**/*._S3_object';
|
||||
const objectFiles = await plugins.smartfile.fs.listFileTree(bucketPath, objectPattern);
|
||||
// List all object files recursively with filter
|
||||
const entries = await plugins.smartfs
|
||||
.directory(bucketPath)
|
||||
.recursive()
|
||||
.filter((entry) => entry.name.endsWith('._S3_object'))
|
||||
.list();
|
||||
|
||||
// Convert file paths to keys
|
||||
let keys = objectFiles.map((filePath) => {
|
||||
const relativePath = plugins.path.relative(bucketPath, filePath);
|
||||
let keys = entries.map((entry) => {
|
||||
const relativePath = plugins.path.relative(bucketPath, entry.path);
|
||||
const key = this.decodeKey(relativePath.replace(/\._S3_object$/, ''));
|
||||
return key;
|
||||
});
|
||||
@@ -226,7 +233,7 @@ export class FilesystemStore {
|
||||
const md5Path = `${objectPath}.md5`;
|
||||
|
||||
const [stats, metadata, md5] = await Promise.all([
|
||||
plugins.smartfile.fs.stat(objectPath),
|
||||
plugins.smartfs.file(objectPath).stat(),
|
||||
this.readMetadata(metadataPath),
|
||||
this.readMD5(objectPath, md5Path),
|
||||
]);
|
||||
@@ -245,7 +252,7 @@ export class FilesystemStore {
|
||||
*/
|
||||
public async objectExists(bucket: string, key: string): Promise<boolean> {
|
||||
const objectPath = this.getObjectPath(bucket, key);
|
||||
return plugins.smartfile.fs.fileExists(objectPath);
|
||||
return plugins.smartfs.file(objectPath).exists();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -265,14 +272,15 @@ export class FilesystemStore {
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
await plugins.fs.promises.mkdir(plugins.path.dirname(objectPath), { recursive: true });
|
||||
const parentDir = plugins.path.dirname(objectPath);
|
||||
await plugins.smartfs.directory(parentDir).recursive().create();
|
||||
|
||||
// Write with MD5 calculation
|
||||
const result = await this.writeStreamWithMD5(stream, objectPath);
|
||||
|
||||
// Save metadata
|
||||
const metadataPath = `${objectPath}.metadata.json`;
|
||||
await plugins.fs.promises.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
|
||||
await plugins.smartfs.file(metadataPath).write(JSON.stringify(metadata, null, 2));
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -293,14 +301,50 @@ export class FilesystemStore {
|
||||
|
||||
const info = await this.getObjectInfo(bucket, key);
|
||||
|
||||
// Create read stream with optional range (using native fs for range support)
|
||||
const stream = range
|
||||
? plugins.fs.createReadStream(objectPath, { start: range.start, end: range.end })
|
||||
: plugins.fs.createReadStream(objectPath);
|
||||
// Get Web ReadableStream from smartfs
|
||||
const webStream = await plugins.smartfs.file(objectPath).readStream();
|
||||
|
||||
// Convert Web Stream to Node.js Readable stream
|
||||
let nodeStream = Readable.fromWeb(webStream as any);
|
||||
|
||||
// Handle range requests if needed
|
||||
if (range) {
|
||||
// For range requests, we need to skip bytes and limit output
|
||||
let bytesRead = 0;
|
||||
const rangeStart = range.start;
|
||||
const rangeEnd = range.end;
|
||||
|
||||
nodeStream = nodeStream.pipe(new (require('stream').Transform)({
|
||||
transform(chunk: Buffer, encoding, callback) {
|
||||
const chunkStart = bytesRead;
|
||||
const chunkEnd = bytesRead + chunk.length - 1;
|
||||
bytesRead += chunk.length;
|
||||
|
||||
// Skip chunks before range
|
||||
if (chunkEnd < rangeStart) {
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop after range
|
||||
if (chunkStart > rangeEnd) {
|
||||
this.end();
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
// Slice chunk to fit range
|
||||
const sliceStart = Math.max(0, rangeStart - chunkStart);
|
||||
const sliceEnd = Math.min(chunk.length, rangeEnd - chunkStart + 1);
|
||||
|
||||
callback(null, chunk.slice(sliceStart, sliceEnd));
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
content: stream,
|
||||
content: nodeStream,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -314,9 +358,9 @@ export class FilesystemStore {
|
||||
|
||||
// S3 doesn't throw error if object doesn't exist
|
||||
await Promise.all([
|
||||
plugins.smartfile.fs.remove(objectPath).catch(() => {}),
|
||||
plugins.smartfile.fs.remove(metadataPath).catch(() => {}),
|
||||
plugins.smartfile.fs.remove(md5Path).catch(() => {}),
|
||||
plugins.smartfs.file(objectPath).delete().catch(() => {}),
|
||||
plugins.smartfs.file(metadataPath).delete().catch(() => {}),
|
||||
plugins.smartfs.file(md5Path).delete().catch(() => {}),
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -345,30 +389,31 @@ export class FilesystemStore {
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
await plugins.fs.promises.mkdir(plugins.path.dirname(destObjectPath), { recursive: true });
|
||||
const parentDir = plugins.path.dirname(destObjectPath);
|
||||
await plugins.smartfs.directory(parentDir).recursive().create();
|
||||
|
||||
// Copy object file
|
||||
await plugins.smartfile.fs.copy(srcObjectPath, destObjectPath);
|
||||
await plugins.smartfs.file(srcObjectPath).copy(destObjectPath);
|
||||
|
||||
// Handle metadata
|
||||
if (metadataDirective === 'COPY') {
|
||||
// Copy metadata
|
||||
const srcMetadataPath = `${srcObjectPath}.metadata.json`;
|
||||
const destMetadataPath = `${destObjectPath}.metadata.json`;
|
||||
await plugins.smartfile.fs.copy(srcMetadataPath, destMetadataPath).catch(() => {});
|
||||
await plugins.smartfs.file(srcMetadataPath).copy(destMetadataPath).catch(() => {});
|
||||
} else if (newMetadata) {
|
||||
// Replace with new metadata
|
||||
const destMetadataPath = `${destObjectPath}.metadata.json`;
|
||||
await plugins.fs.promises.writeFile(destMetadataPath, JSON.stringify(newMetadata, null, 2));
|
||||
await plugins.smartfs.file(destMetadataPath).write(JSON.stringify(newMetadata, null, 2));
|
||||
}
|
||||
|
||||
// Copy MD5
|
||||
const srcMD5Path = `${srcObjectPath}.md5`;
|
||||
const destMD5Path = `${destObjectPath}.md5`;
|
||||
await plugins.smartfile.fs.copy(srcMD5Path, destMD5Path).catch(() => {});
|
||||
await plugins.smartfs.file(srcMD5Path).copy(destMD5Path).catch(() => {});
|
||||
|
||||
// Get result info
|
||||
const stats = await plugins.smartfile.fs.stat(destObjectPath);
|
||||
const stats = await plugins.smartfs.file(destObjectPath).stat();
|
||||
const md5 = await this.readMD5(destObjectPath, destMD5Path);
|
||||
|
||||
return { size: stats.size, md5 };
|
||||
@@ -432,25 +477,41 @@ export class FilesystemStore {
|
||||
const hash = plugins.crypto.createHash('md5');
|
||||
let totalSize = 0;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const output = plugins.fs.createWriteStream(destPath);
|
||||
return new Promise(async (resolve, reject) => {
|
||||
// Get Web WritableStream from smartfs
|
||||
const webWriteStream = await plugins.smartfs.file(destPath).writeStream();
|
||||
const writer = webWriteStream.getWriter();
|
||||
|
||||
input.on('data', (chunk: Buffer) => {
|
||||
// Read from Node.js stream and write to Web stream
|
||||
input.on('data', async (chunk: Buffer) => {
|
||||
hash.update(chunk);
|
||||
totalSize += chunk.length;
|
||||
|
||||
try {
|
||||
await writer.write(new Uint8Array(chunk));
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
input.on('error', reject);
|
||||
output.on('error', reject);
|
||||
input.on('error', (err) => {
|
||||
writer.abort(err);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
input.pipe(output).on('finish', async () => {
|
||||
input.on('end', async () => {
|
||||
try {
|
||||
await writer.close();
|
||||
const md5 = hash.digest('hex');
|
||||
|
||||
// Save MD5 to separate file
|
||||
const md5Path = `${destPath}.md5`;
|
||||
await plugins.fs.promises.writeFile(md5Path, md5);
|
||||
await plugins.smartfs.file(md5Path).write(md5);
|
||||
|
||||
resolve({ size: totalSize, md5 });
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -461,22 +522,28 @@ export class FilesystemStore {
|
||||
private async readMD5(objectPath: string, md5Path: string): Promise<string> {
|
||||
try {
|
||||
// Try to read cached MD5
|
||||
const md5 = await plugins.smartfile.fs.toStringSync(md5Path);
|
||||
const md5 = await plugins.smartfs.file(md5Path).encoding('utf8').read() as string;
|
||||
return md5.trim();
|
||||
} catch (err) {
|
||||
// Calculate MD5 if not cached
|
||||
return new Promise((resolve, reject) => {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const hash = plugins.crypto.createHash('md5');
|
||||
const stream = plugins.fs.createReadStream(objectPath);
|
||||
|
||||
stream.on('data', (chunk: Buffer) => hash.update(chunk));
|
||||
stream.on('end', async () => {
|
||||
try {
|
||||
const webStream = await plugins.smartfs.file(objectPath).readStream();
|
||||
const nodeStream = Readable.fromWeb(webStream as any);
|
||||
|
||||
nodeStream.on('data', (chunk: Buffer) => hash.update(chunk));
|
||||
nodeStream.on('end', async () => {
|
||||
const md5 = hash.digest('hex');
|
||||
// Cache it
|
||||
await plugins.fs.promises.writeFile(md5Path, md5);
|
||||
await plugins.smartfs.file(md5Path).write(md5);
|
||||
resolve(md5);
|
||||
});
|
||||
stream.on('error', reject);
|
||||
nodeStream.on('error', reject);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -486,7 +553,7 @@ export class FilesystemStore {
|
||||
*/
|
||||
private async readMetadata(metadataPath: string): Promise<Record<string, string>> {
|
||||
try {
|
||||
const content = await plugins.smartfile.fs.toStringSync(metadataPath);
|
||||
const content = await plugins.smartfs.file(metadataPath).encoding('utf8').read() as string;
|
||||
return JSON.parse(content);
|
||||
} catch (err) {
|
||||
return {};
|
||||
|
||||
@@ -3,17 +3,19 @@ import * as path from 'path';
|
||||
import * as http from 'http';
|
||||
import * as crypto from 'crypto';
|
||||
import * as url from 'url';
|
||||
import * as fs from 'fs';
|
||||
|
||||
export { path, http, crypto, url, fs };
|
||||
export { path, http, crypto, url };
|
||||
|
||||
// @push.rocks scope
|
||||
import * as smartbucket from '@push.rocks/smartbucket';
|
||||
import * as smartfile from '@push.rocks/smartfile';
|
||||
import { SmartFs, SmartFsProviderNode } from '@push.rocks/smartfs';
|
||||
import * as smartpath from '@push.rocks/smartpath';
|
||||
import { SmartXml } from '@push.rocks/smartxml';
|
||||
|
||||
export { smartbucket, smartfile, smartpath, SmartXml };
|
||||
// Create SmartFs instance with Node.js provider
|
||||
export const smartfs = new SmartFs(new SmartFsProviderNode());
|
||||
|
||||
export { smartbucket, smartpath, SmartXml };
|
||||
|
||||
// @tsclass scope
|
||||
import * as tsclass from '@tsclass/tsclass';
|
||||
|
||||
Reference in New Issue
Block a user