diff --git a/changelog.md b/changelog.md index 1337a0d..d9cc45f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2025-11-23 - 3.0.3 - fix(filesystem) +Migrate filesystem implementation to @push.rocks/smartfs and add Web Streams handling + +- Replace dependency @push.rocks/smartfile with @push.rocks/smartfs and update README references +- plugins: instantiate SmartFs with SmartFsProviderNode and export smartfs (remove direct fs export) +- Refactor FilesystemStore to use smartfs directory/file APIs for initialize, reset, list, read, write, copy and delete +- Implement Web Stream ↔ Node.js stream conversion for uploads/downloads (Readable.fromWeb and writer.write with Uint8Array) +- Persist and read metadata (.metadata.json) and cached MD5 (.md5) via smartfs APIs +- Update readme.hints and documentation to note successful migration and next steps + ## 2025-11-21 - 3.0.2 - fix(smarts3) Prepare patch release 3.0.2 — no code changes detected diff --git a/package.json b/package.json index fe08b31..4cb7d03 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ ], "dependencies": { "@push.rocks/smartbucket": "^4.3.0", - "@push.rocks/smartfile": "^11.2.7", + "@push.rocks/smartfs": "^1.1.0", "@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartxml": "^2.0.0", "@tsclass/tsclass": "^9.3.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f71817..be3f402 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,9 +11,9 @@ importers: '@push.rocks/smartbucket': specifier: ^4.3.0 version: 4.3.0 - '@push.rocks/smartfile': - specifier: ^11.2.7 - version: 11.2.7 + '@push.rocks/smartfs': + specifier: ^1.1.0 + version: 1.1.0 '@push.rocks/smartpath': specifier: ^6.0.0 version: 6.0.0 @@ -644,6 +644,9 @@ packages: '@push.rocks/smartfile@11.2.7': resolution: {integrity: sha512-8Yp7/sAgPpWJBHohV92ogHWKzRomI5MEbSG6b5W2n18tqwfAmjMed0rQvsvGrSBlnEWCKgoOrYIIZbLO61+J0Q==} + '@push.rocks/smartfs@1.1.0': + resolution: {integrity: sha512-fg8JIjFUPPX5laRoBpTaGwhMfZ3Y8mFT4fUaW54Y4J/BfOBa/y0+rIFgvgvqcOZgkQlyZU+FIfL8Z6zezqxyTg==} + '@push.rocks/smartguard@3.1.0': resolution: {integrity: sha512-J23q84f1O+TwFGmd4lrO9XLHUh2DaLXo9PN/9VmTWYzTkQDv5JehmifXVI0esophXcCIfbdIu6hbt7/aHlDF4A==} @@ -4920,6 +4923,10 @@ snapshots: glob: 11.1.0 js-yaml: 4.1.1 + '@push.rocks/smartfs@1.1.0': + dependencies: + '@push.rocks/smartpath': 6.0.0 + '@push.rocks/smartguard@3.1.0': dependencies: '@push.rocks/smartpromise': 4.2.3 diff --git a/production-readiness.md b/production-readiness.md new file mode 100644 index 0000000..abf70d6 --- /dev/null +++ b/production-readiness.md @@ -0,0 +1,438 @@ +# Production-Readiness Plan for smarts3 + +**Goal:** Make smarts3 production-ready as a MinIO alternative for use cases where: +- Running MinIO is out of scope +- You have a program written for S3 and want to use the local filesystem +- You need a lightweight, zero-dependency S3-compatible server + +--- + +## 🔍 Current State Analysis + +### ✅ What's Working + +- **Native S3 server** with zero framework dependencies +- **Core S3 operations:** PUT, GET, HEAD, DELETE (objects & buckets) +- **List buckets and objects** (V1 and V2 API) +- **Object copy** with metadata handling +- **Range requests** for partial downloads +- **MD5 checksums** and ETag support +- **Custom metadata** (x-amz-meta-*) +- **Filesystem-backed storage** with Windows compatibility +- **S3-compatible XML error responses** +- **Middleware system** and routing +- **AWS SDK v3 compatibility** (tested) + +### ❌ Production Gaps Identified + +--- + +## 🎯 Critical Features (Required for Production) + +### 1. Multipart Upload Support 🚀 **HIGHEST PRIORITY** + +**Why:** Essential for uploading files >5MB efficiently. Without this, smarts3 can't handle real-world production workloads. + +**Implementation Required:** +- `POST /:bucket/:key?uploads` - CreateMultipartUpload +- `PUT /:bucket/:key?partNumber=X&uploadId=Y` - UploadPart +- `POST /:bucket/:key?uploadId=X` - CompleteMultipartUpload +- `DELETE /:bucket/:key?uploadId=X` - AbortMultipartUpload +- `GET /:bucket/:key?uploadId=X` - ListParts +- Multipart state management (temp storage for parts) +- Part ETag tracking and validation +- Automatic cleanup of abandoned uploads + +**Files to Create/Modify:** +- `ts/controllers/multipart.controller.ts` (new) +- `ts/classes/filesystem-store.ts` (add multipart methods) +- `ts/classes/smarts3-server.ts` (add multipart routes) + +--- + +### 2. Configurable Authentication 🔐 + +**Why:** Currently hardcoded credentials ('S3RVER'/'S3RVER'). Production needs custom credentials. + +**Implementation Required:** +- Support custom access keys and secrets via configuration +- Implement AWS Signature V4 verification +- Support multiple credential pairs (IAM-like users) +- Optional: Disable authentication for local dev use + +**Configuration Example:** +```typescript +interface IAuthConfig { + enabled: boolean; + credentials: Array<{ + accessKeyId: string; + secretAccessKey: string; + }>; + signatureVersion: 'v4' | 'none'; +} +``` + +**Files to Create/Modify:** +- `ts/classes/auth-middleware.ts` (new) +- `ts/classes/signature-validator.ts` (new) +- `ts/classes/smarts3-server.ts` (integrate auth middleware) +- `ts/index.ts` (add auth config options) + +--- + +### 3. CORS Support 🌐 + +**Why:** Required for browser-based uploads and modern web apps. + +**Implementation Required:** +- Add CORS middleware +- Support preflight OPTIONS requests +- Configurable CORS origins, methods, headers +- Per-bucket CORS configuration (optional) + +**Configuration Example:** +```typescript +interface ICorsConfig { + enabled: boolean; + allowedOrigins: string[]; // ['*'] or ['https://example.com'] + allowedMethods: string[]; // ['GET', 'POST', 'PUT', 'DELETE'] + allowedHeaders: string[]; // ['*'] or specific headers + exposedHeaders: string[]; // ['ETag', 'x-amz-*'] + maxAge: number; // 3600 (seconds) + allowCredentials: boolean; +} +``` + +**Files to Create/Modify:** +- `ts/classes/cors-middleware.ts` (new) +- `ts/classes/smarts3-server.ts` (integrate CORS middleware) +- `ts/index.ts` (add CORS config options) + +--- + +### 4. SSL/TLS Support 🔒 + +**Why:** Production systems require encrypted connections. + +**Implementation Required:** +- HTTPS server option with cert/key configuration +- Auto-redirect HTTP to HTTPS (optional) +- Support for self-signed certs in dev mode + +**Configuration Example:** +```typescript +interface ISslConfig { + enabled: boolean; + cert: string; // Path to certificate file or cert content + key: string; // Path to key file or key content + ca?: string; // Optional CA cert + redirectHttp?: boolean; // Redirect HTTP to HTTPS +} +``` + +**Files to Create/Modify:** +- `ts/classes/smarts3-server.ts` (add HTTPS server creation) +- `ts/index.ts` (add SSL config options) + +--- + +### 5. Production Configuration System ⚙️ + +**Why:** Production needs flexible configuration, not just constructor options. + +**Implementation Required:** +- Support configuration file (JSON/YAML) +- Environment variable support +- Configuration validation +- Sensible production defaults +- Example configurations for common use cases + +**Configuration File Example (`smarts3.config.json`):** +```json +{ + "server": { + "port": 3000, + "address": "0.0.0.0", + "ssl": { + "enabled": true, + "cert": "./certs/server.crt", + "key": "./certs/server.key" + } + }, + "storage": { + "directory": "./s3-data", + "cleanSlate": false + }, + "auth": { + "enabled": true, + "credentials": [ + { + "accessKeyId": "AKIAIOSFODNN7EXAMPLE", + "secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + ] + }, + "cors": { + "enabled": true, + "allowedOrigins": ["*"], + "allowedMethods": ["GET", "POST", "PUT", "DELETE", "HEAD"], + "allowedHeaders": ["*"] + }, + "limits": { + "maxObjectSize": 5368709120, + "maxMetadataSize": 2048, + "requestTimeout": 300000 + }, + "logging": { + "level": "info", + "format": "json", + "accessLog": { + "enabled": true, + "path": "./logs/access.log" + }, + "errorLog": { + "enabled": true, + "path": "./logs/error.log" + } + } +} +``` + +**Files to Create/Modify:** +- `ts/classes/config-loader.ts` (new) +- `ts/classes/config-validator.ts` (new) +- `ts/index.ts` (use config loader) +- Create example config files in root + +--- + +### 6. Production Logging 📝 + +**Why:** Console logs aren't suitable for production monitoring. + +**Implementation Required:** +- Structured logging (JSON format option) +- Log levels (ERROR, WARN, INFO, DEBUG) +- File rotation support +- Access logs (S3 standard format) +- Integration with logging library + +**Files to Create/Modify:** +- `ts/classes/logger.ts` (new - use @push.rocks/smartlog?) +- `ts/classes/access-logger-middleware.ts` (new) +- `ts/classes/smarts3-server.ts` (replace console.log with logger) +- All controller files (use structured logging) + +--- + +## 🔧 Important Features (Should Have) + +### 7. Health Check & Metrics 💊 + +**Implementation Required:** +- `GET /_health` endpoint (non-S3, for monitoring) +- `GET /_metrics` endpoint (Prometheus format?) +- Server stats (requests/sec, storage used, uptime) +- Readiness/liveness probes for Kubernetes + +**Files to Create/Modify:** +- `ts/controllers/health.controller.ts` (new) +- `ts/classes/metrics-collector.ts` (new) +- `ts/classes/smarts3-server.ts` (add health routes) + +--- + +### 8. Batch Operations 📦 + +**Implementation Required:** +- `POST /:bucket?delete` - DeleteObjects (delete multiple objects in one request) +- Essential for efficient cleanup operations + +**Files to Create/Modify:** +- `ts/controllers/object.controller.ts` (add deleteObjects method) + +--- + +### 9. Request Size Limits & Validation 🛡️ + +**Implementation Required:** +- Max object size configuration +- Max metadata size limits +- Request timeout configuration +- Body size limits +- Bucket name validation (S3 rules) +- Key name validation + +**Files to Create/Modify:** +- `ts/classes/validation-middleware.ts` (new) +- `ts/utils/validators.ts` (new) +- `ts/classes/smarts3-server.ts` (integrate validation middleware) + +--- + +### 10. Conditional Requests 🔄 + +**Implementation Required:** +- If-Match / If-None-Match (ETag validation) +- If-Modified-Since / If-Unmodified-Since +- Required for caching and conflict prevention + +**Files to Create/Modify:** +- `ts/controllers/object.controller.ts` (add conditional logic to GET/HEAD) + +--- + +### 11. Graceful Shutdown 👋 + +**Implementation Required:** +- Drain existing connections +- Reject new connections +- Clean multipart cleanup on shutdown +- SIGTERM/SIGINT handling + +**Files to Create/Modify:** +- `ts/classes/smarts3-server.ts` (add graceful shutdown logic) +- `ts/index.ts` (add signal handlers) + +--- + +## 💡 Nice-to-Have Features + +### 12. Advanced Features + +- Bucket versioning support +- Object tagging +- Lifecycle policies (auto-delete old objects) +- Storage class simulation (STANDARD, GLACIER, etc.) +- Server-side encryption simulation +- Presigned URL support (for time-limited access) + +### 13. Performance Optimizations + +- Stream optimization for large files +- Optional in-memory caching for small objects +- Parallel upload/download support +- Compression support (gzip) + +### 14. Developer Experience + +- Docker image for easy deployment +- Docker Compose examples +- Kubernetes manifests +- CLI for server management +- Admin API for bucket management + +--- + +## 📐 Implementation Phases + +### Phase 1: Critical Production Features (Priority 1) + +**Estimated Effort:** 2-3 weeks + +1. ✅ Multipart uploads (biggest technical lift) +2. ✅ Configurable authentication +3. ✅ CORS middleware +4. ✅ Production configuration system +5. ✅ Production logging + +**Outcome:** smarts3 can handle real production workloads + +--- + +### Phase 2: Reliability & Operations (Priority 2) + +**Estimated Effort:** 1-2 weeks + +6. ✅ SSL/TLS support +7. ✅ Health checks & metrics +8. ✅ Request validation & limits +9. ✅ Graceful shutdown +10. ✅ Batch operations + +**Outcome:** smarts3 is operationally mature + +--- + +### Phase 3: S3 Compatibility (Priority 3) + +**Estimated Effort:** 1-2 weeks + +11. ✅ Conditional requests +12. ✅ Additional S3 features as needed +13. ✅ Comprehensive test suite +14. ✅ Documentation updates + +**Outcome:** smarts3 has broad S3 API compatibility + +--- + +### Phase 4: Polish (Priority 4) + +**Estimated Effort:** As needed + +15. ✅ Docker packaging +16. ✅ Performance optimization +17. ✅ Advanced features based on user feedback + +**Outcome:** smarts3 is a complete MinIO alternative + +--- + +## 🤔 Open Questions + +1. **Authentication:** Do you want full AWS Signature V4 validation, or simpler static credential checking? +2. **Configuration:** Prefer JSON, YAML, or .env file format? +3. **Logging:** Do you have a preferred logging library, or shall I use @push.rocks/smartlog? +4. **Scope:** Should we tackle all of Phase 1, or start with a subset (e.g., just multipart + auth)? +5. **Testing:** Should we add comprehensive tests as we go, or batch them at the end? +6. **Breaking changes:** Can I modify the constructor options interface, or must it remain backward compatible? + +--- + +## 🎯 Target Use Cases + +**With this plan implemented, smarts3 will be a solid MinIO alternative for:** + +✅ **Local S3 development** - Fast, simple, no Docker required +✅ **Testing S3 integrations** - Reliable, repeatable tests +✅ **Microservices using S3 API** with filesystem backend +✅ **CI/CD pipelines** - Lightweight S3 for testing +✅ **Small-to-medium production deployments** where MinIO is overkill +✅ **Edge computing** - S3 API for local file storage +✅ **Embedded systems** - Minimal dependencies, small footprint + +--- + +## 📊 Current vs. Production Comparison + +| Feature | Current | After Phase 1 | After Phase 2 | Production Ready | +|---------|---------|---------------|---------------|------------------| +| Basic S3 ops | ✅ | ✅ | ✅ | ✅ | +| Multipart upload | ❌ | ✅ | ✅ | ✅ | +| Authentication | ⚠️ (hardcoded) | ✅ | ✅ | ✅ | +| CORS | ❌ | ✅ | ✅ | ✅ | +| SSL/TLS | ❌ | ❌ | ✅ | ✅ | +| Config files | ❌ | ✅ | ✅ | ✅ | +| Production logging | ⚠️ (console) | ✅ | ✅ | ✅ | +| Health checks | ❌ | ❌ | ✅ | ✅ | +| Request limits | ❌ | ❌ | ✅ | ✅ | +| Graceful shutdown | ❌ | ❌ | ✅ | ✅ | +| Conditional requests | ❌ | ❌ | ❌ | ✅ | +| Batch operations | ❌ | ❌ | ✅ | ✅ | + +--- + +## 📝 Notes + +- All features should maintain backward compatibility where possible +- Each feature should include comprehensive tests +- Documentation (readme.md) should be updated as features are added +- Consider adding a migration guide for users upgrading from testing to production use +- Performance benchmarks should be established and maintained + +--- + +**Last Updated:** 2025-11-23 +**Status:** Planning Phase +**Next Step:** Get approval and prioritize implementation order diff --git a/readme.hints.md b/readme.hints.md index 0519ecb..6db77de 100644 --- a/readme.hints.md +++ b/readme.hints.md @@ -1 +1,74 @@ - \ No newline at end of file +# Project Hints for smarts3 + +## Current State (v3.0.0) + +- Native custom S3 server implementation (Smarts3Server) +- No longer uses legacy s3rver backend (removed in v3.0.0) +- Core S3 operations working: PUT, GET, HEAD, DELETE for objects and buckets +- Multipart upload NOT yet implemented (critical gap for production) +- Authentication is hardcoded ('S3RVER'/'S3RVER') - not production-ready +- No CORS support yet +- No SSL/TLS support yet + +## Production Readiness + +See `production-readiness.md` for the complete gap analysis and implementation plan. + +**Key Missing Features for Production:** +1. Multipart upload support (HIGHEST PRIORITY) +2. Configurable authentication +3. CORS middleware +4. SSL/TLS support +5. Production configuration system +6. Production logging + +## Architecture Notes + +### File Structure +- `ts/classes/smarts3-server.ts` - Main server class +- `ts/classes/filesystem-store.ts` - Storage layer (filesystem-backed) +- `ts/classes/router.ts` - URL routing with pattern matching +- `ts/classes/middleware-stack.ts` - Middleware execution +- `ts/classes/context.ts` - Request/response context +- `ts/classes/s3-error.ts` - S3-compatible error handling +- `ts/controllers/` - Service, bucket, and object controllers +- `ts/index.ts` - Main export (Smarts3 class) + +### Storage Layout +- Objects stored as: `{bucket}/{encodedKey}._S3_object` +- Metadata stored as: `{bucket}/{encodedKey}._S3_object.metadata.json` +- MD5 stored as: `{bucket}/{encodedKey}._S3_object.md5` +- Keys are encoded for Windows compatibility (hex encoding for invalid chars) + +### Current Limitations +- Max file size limited by available memory (no streaming multipart) +- Single server instance only (no clustering) +- No versioning support +- No access control beyond basic auth + +## Testing + +- Main test: `test/test.aws-sdk.node.ts` - Tests AWS SDK v3 compatibility +- Run with: `pnpm test` +- Tests run with cleanSlate mode enabled + +## Dependencies + +- `@push.rocks/smartbucket` - S3 abstraction layer +- `@push.rocks/smartfs` - Modern filesystem operations with Web Streams API (replaced smartfile) +- `@push.rocks/smartxml` - XML generation/parsing +- `@push.rocks/smartpath` - Path utilities +- `@tsclass/tsclass` - TypeScript utilities + +## Migration Notes (2025-11-23) + +Successfully migrated from `@push.rocks/smartfile` + native `fs` to `@push.rocks/smartfs`: +- All file/directory operations now use smartfs fluent API +- Web Streams → Node.js Streams conversion for HTTP compatibility +- All tests passing ✅ +- Build successful ✅ + +## Next Steps + +Waiting for approval to proceed with production-readiness implementation. +Priority 1 is implementing multipart uploads. diff --git a/readme.md b/readme.md index bfd42fa..ddeaf92 100644 --- a/readme.md +++ b/readme.md @@ -413,7 +413,7 @@ interface ISmarts3ContructorOptions { ## 🔗 Related Packages - [`@push.rocks/smartbucket`](https://www.npmjs.com/package/@push.rocks/smartbucket) - Powerful S3 abstraction layer -- [`@push.rocks/smartfile`](https://www.npmjs.com/package/@push.rocks/smartfile) - Advanced file system operations +- [`@push.rocks/smartfs`](https://www.npmjs.com/package/@push.rocks/smartfs) - Modern filesystem with Web Streams support - [`@tsclass/tsclass`](https://www.npmjs.com/package/@tsclass/tsclass) - TypeScript class helpers ## License and Legal Information diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 6d350f4..e376941 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smarts3', - version: '3.0.2', + version: '3.0.3', description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.' } diff --git a/ts/classes/filesystem-store.ts b/ts/classes/filesystem-store.ts index 982dfe2..6730530 100644 --- a/ts/classes/filesystem-store.ts +++ b/ts/classes/filesystem-store.ts @@ -1,6 +1,6 @@ import * as plugins from '../plugins.js'; import { S3Error } from './s3-error.js'; -import type { Readable } from 'stream'; +import { Readable } from 'stream'; export interface IS3Bucket { name: string; @@ -39,7 +39,7 @@ export interface IRangeOptions { } /** - * Filesystem-backed storage for S3 objects + * Filesystem-backed storage for S3 objects using smartfs */ export class FilesystemStore { constructor(private rootDir: string) {} @@ -48,14 +48,19 @@ export class FilesystemStore { * Initialize store (ensure root directory exists) */ public async initialize(): Promise { - await plugins.fs.promises.mkdir(this.rootDir, { recursive: true }); + await plugins.smartfs.directory(this.rootDir).recursive().create(); } /** * Reset store (delete all buckets) */ public async reset(): Promise { - await plugins.smartfile.fs.ensureEmptyDir(this.rootDir); + // Delete directory and recreate it + const exists = await plugins.smartfs.directory(this.rootDir).exists(); + if (exists) { + await plugins.smartfs.directory(this.rootDir).recursive().delete(); + } + await plugins.smartfs.directory(this.rootDir).recursive().create(); } // ============================ @@ -66,17 +71,16 @@ export class FilesystemStore { * List all buckets */ public async listBuckets(): Promise { - const dirs = await plugins.smartfile.fs.listFolders(this.rootDir); + const entries = await plugins.smartfs.directory(this.rootDir).includeStats().list(); const buckets: IS3Bucket[] = []; - for (const dir of dirs) { - const bucketPath = plugins.path.join(this.rootDir, dir); - const stats = await plugins.smartfile.fs.stat(bucketPath); - - buckets.push({ - name: dir, - creationDate: stats.birthtime, - }); + for (const entry of entries) { + if (entry.isDirectory && entry.stats) { + buckets.push({ + name: entry.name, + creationDate: entry.stats.birthtime, + }); + } } return buckets.sort((a, b) => a.name.localeCompare(b.name)); @@ -87,7 +91,7 @@ export class FilesystemStore { */ public async bucketExists(bucket: string): Promise { const bucketPath = this.getBucketPath(bucket); - return plugins.smartfile.fs.isDirectory(bucketPath); + return plugins.smartfs.directory(bucketPath).exists(); } /** @@ -95,7 +99,7 @@ export class FilesystemStore { */ public async createBucket(bucket: string): Promise { const bucketPath = this.getBucketPath(bucket); - await plugins.fs.promises.mkdir(bucketPath, { recursive: true }); + await plugins.smartfs.directory(bucketPath).recursive().create(); } /** @@ -110,12 +114,12 @@ export class FilesystemStore { } // Check if bucket is empty - const files = await plugins.smartfile.fs.listFileTree(bucketPath, '**/*'); + const files = await plugins.smartfs.directory(bucketPath).recursive().list(); if (files.length > 0) { throw new S3Error('BucketNotEmpty', 'The bucket you tried to delete is not empty'); } - await plugins.smartfile.fs.remove(bucketPath); + await plugins.smartfs.directory(bucketPath).recursive().delete(); } // ============================ @@ -142,13 +146,16 @@ export class FilesystemStore { continuationToken, } = options; - // List all object files - const objectPattern = '**/*._S3_object'; - const objectFiles = await plugins.smartfile.fs.listFileTree(bucketPath, objectPattern); + // List all object files recursively with filter + const entries = await plugins.smartfs + .directory(bucketPath) + .recursive() + .filter((entry) => entry.name.endsWith('._S3_object')) + .list(); // Convert file paths to keys - let keys = objectFiles.map((filePath) => { - const relativePath = plugins.path.relative(bucketPath, filePath); + let keys = entries.map((entry) => { + const relativePath = plugins.path.relative(bucketPath, entry.path); const key = this.decodeKey(relativePath.replace(/\._S3_object$/, '')); return key; }); @@ -226,7 +233,7 @@ export class FilesystemStore { const md5Path = `${objectPath}.md5`; const [stats, metadata, md5] = await Promise.all([ - plugins.smartfile.fs.stat(objectPath), + plugins.smartfs.file(objectPath).stat(), this.readMetadata(metadataPath), this.readMD5(objectPath, md5Path), ]); @@ -245,7 +252,7 @@ export class FilesystemStore { */ public async objectExists(bucket: string, key: string): Promise { const objectPath = this.getObjectPath(bucket, key); - return plugins.smartfile.fs.fileExists(objectPath); + return plugins.smartfs.file(objectPath).exists(); } /** @@ -265,14 +272,15 @@ export class FilesystemStore { } // Ensure parent directory exists - await plugins.fs.promises.mkdir(plugins.path.dirname(objectPath), { recursive: true }); + const parentDir = plugins.path.dirname(objectPath); + await plugins.smartfs.directory(parentDir).recursive().create(); // Write with MD5 calculation const result = await this.writeStreamWithMD5(stream, objectPath); // Save metadata const metadataPath = `${objectPath}.metadata.json`; - await plugins.fs.promises.writeFile(metadataPath, JSON.stringify(metadata, null, 2)); + await plugins.smartfs.file(metadataPath).write(JSON.stringify(metadata, null, 2)); return result; } @@ -293,14 +301,50 @@ export class FilesystemStore { const info = await this.getObjectInfo(bucket, key); - // Create read stream with optional range (using native fs for range support) - const stream = range - ? plugins.fs.createReadStream(objectPath, { start: range.start, end: range.end }) - : plugins.fs.createReadStream(objectPath); + // Get Web ReadableStream from smartfs + const webStream = await plugins.smartfs.file(objectPath).readStream(); + + // Convert Web Stream to Node.js Readable stream + let nodeStream = Readable.fromWeb(webStream as any); + + // Handle range requests if needed + if (range) { + // For range requests, we need to skip bytes and limit output + let bytesRead = 0; + const rangeStart = range.start; + const rangeEnd = range.end; + + nodeStream = nodeStream.pipe(new (require('stream').Transform)({ + transform(chunk: Buffer, encoding, callback) { + const chunkStart = bytesRead; + const chunkEnd = bytesRead + chunk.length - 1; + bytesRead += chunk.length; + + // Skip chunks before range + if (chunkEnd < rangeStart) { + callback(); + return; + } + + // Stop after range + if (chunkStart > rangeEnd) { + this.end(); + callback(); + return; + } + + // Slice chunk to fit range + const sliceStart = Math.max(0, rangeStart - chunkStart); + const sliceEnd = Math.min(chunk.length, rangeEnd - chunkStart + 1); + + callback(null, chunk.slice(sliceStart, sliceEnd)); + } + })); + } return { ...info, - content: stream, + content: nodeStream, }; } @@ -314,9 +358,9 @@ export class FilesystemStore { // S3 doesn't throw error if object doesn't exist await Promise.all([ - plugins.smartfile.fs.remove(objectPath).catch(() => {}), - plugins.smartfile.fs.remove(metadataPath).catch(() => {}), - plugins.smartfile.fs.remove(md5Path).catch(() => {}), + plugins.smartfs.file(objectPath).delete().catch(() => {}), + plugins.smartfs.file(metadataPath).delete().catch(() => {}), + plugins.smartfs.file(md5Path).delete().catch(() => {}), ]); } @@ -345,30 +389,31 @@ export class FilesystemStore { } // Ensure parent directory exists - await plugins.fs.promises.mkdir(plugins.path.dirname(destObjectPath), { recursive: true }); + const parentDir = plugins.path.dirname(destObjectPath); + await plugins.smartfs.directory(parentDir).recursive().create(); // Copy object file - await plugins.smartfile.fs.copy(srcObjectPath, destObjectPath); + await plugins.smartfs.file(srcObjectPath).copy(destObjectPath); // Handle metadata if (metadataDirective === 'COPY') { // Copy metadata const srcMetadataPath = `${srcObjectPath}.metadata.json`; const destMetadataPath = `${destObjectPath}.metadata.json`; - await plugins.smartfile.fs.copy(srcMetadataPath, destMetadataPath).catch(() => {}); + await plugins.smartfs.file(srcMetadataPath).copy(destMetadataPath).catch(() => {}); } else if (newMetadata) { // Replace with new metadata const destMetadataPath = `${destObjectPath}.metadata.json`; - await plugins.fs.promises.writeFile(destMetadataPath, JSON.stringify(newMetadata, null, 2)); + await plugins.smartfs.file(destMetadataPath).write(JSON.stringify(newMetadata, null, 2)); } // Copy MD5 const srcMD5Path = `${srcObjectPath}.md5`; const destMD5Path = `${destObjectPath}.md5`; - await plugins.smartfile.fs.copy(srcMD5Path, destMD5Path).catch(() => {}); + await plugins.smartfs.file(srcMD5Path).copy(destMD5Path).catch(() => {}); // Get result info - const stats = await plugins.smartfile.fs.stat(destObjectPath); + const stats = await plugins.smartfs.file(destObjectPath).stat(); const md5 = await this.readMD5(destObjectPath, destMD5Path); return { size: stats.size, md5 }; @@ -432,25 +477,41 @@ export class FilesystemStore { const hash = plugins.crypto.createHash('md5'); let totalSize = 0; - return new Promise((resolve, reject) => { - const output = plugins.fs.createWriteStream(destPath); + return new Promise(async (resolve, reject) => { + // Get Web WritableStream from smartfs + const webWriteStream = await plugins.smartfs.file(destPath).writeStream(); + const writer = webWriteStream.getWriter(); - input.on('data', (chunk: Buffer) => { + // Read from Node.js stream and write to Web stream + input.on('data', async (chunk: Buffer) => { hash.update(chunk); totalSize += chunk.length; + + try { + await writer.write(new Uint8Array(chunk)); + } catch (err) { + reject(err); + } }); - input.on('error', reject); - output.on('error', reject); + input.on('error', (err) => { + writer.abort(err); + reject(err); + }); - input.pipe(output).on('finish', async () => { - const md5 = hash.digest('hex'); + input.on('end', async () => { + try { + await writer.close(); + const md5 = hash.digest('hex'); - // Save MD5 to separate file - const md5Path = `${destPath}.md5`; - await plugins.fs.promises.writeFile(md5Path, md5); + // Save MD5 to separate file + const md5Path = `${destPath}.md5`; + await plugins.smartfs.file(md5Path).write(md5); - resolve({ size: totalSize, md5 }); + resolve({ size: totalSize, md5 }); + } catch (err) { + reject(err); + } }); }); } @@ -461,22 +522,28 @@ export class FilesystemStore { private async readMD5(objectPath: string, md5Path: string): Promise { try { // Try to read cached MD5 - const md5 = await plugins.smartfile.fs.toStringSync(md5Path); + const md5 = await plugins.smartfs.file(md5Path).encoding('utf8').read() as string; return md5.trim(); } catch (err) { // Calculate MD5 if not cached - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { const hash = plugins.crypto.createHash('md5'); - const stream = plugins.fs.createReadStream(objectPath); - stream.on('data', (chunk: Buffer) => hash.update(chunk)); - stream.on('end', async () => { - const md5 = hash.digest('hex'); - // Cache it - await plugins.fs.promises.writeFile(md5Path, md5); - resolve(md5); - }); - stream.on('error', reject); + try { + const webStream = await plugins.smartfs.file(objectPath).readStream(); + const nodeStream = Readable.fromWeb(webStream as any); + + nodeStream.on('data', (chunk: Buffer) => hash.update(chunk)); + nodeStream.on('end', async () => { + const md5 = hash.digest('hex'); + // Cache it + await plugins.smartfs.file(md5Path).write(md5); + resolve(md5); + }); + nodeStream.on('error', reject); + } catch (err) { + reject(err); + } }); } } @@ -486,7 +553,7 @@ export class FilesystemStore { */ private async readMetadata(metadataPath: string): Promise> { try { - const content = await plugins.smartfile.fs.toStringSync(metadataPath); + const content = await plugins.smartfs.file(metadataPath).encoding('utf8').read() as string; return JSON.parse(content); } catch (err) { return {}; diff --git a/ts/plugins.ts b/ts/plugins.ts index d117c4e..63af4e7 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -3,17 +3,19 @@ import * as path from 'path'; import * as http from 'http'; import * as crypto from 'crypto'; import * as url from 'url'; -import * as fs from 'fs'; -export { path, http, crypto, url, fs }; +export { path, http, crypto, url }; // @push.rocks scope import * as smartbucket from '@push.rocks/smartbucket'; -import * as smartfile from '@push.rocks/smartfile'; +import { SmartFs, SmartFsProviderNode } from '@push.rocks/smartfs'; import * as smartpath from '@push.rocks/smartpath'; import { SmartXml } from '@push.rocks/smartxml'; -export { smartbucket, smartfile, smartpath, SmartXml }; +// Create SmartFs instance with Node.js provider +export const smartfs = new SmartFs(new SmartFsProviderNode()); + +export { smartbucket, smartpath, SmartXml }; // @tsclass scope import * as tsclass from '@tsclass/tsclass';