diff --git a/changelog.md b/changelog.md index 5867e8e..a4e74a8 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog +## 2025-11-23 - 5.1.0 - feat(multipart) +Implement full multipart upload support with persistent manager, periodic cleanup, and API integration + +- Add IMultipartConfig to server config with defaults (expirationDays: 7, cleanupIntervalMinutes: 60) and merge into existing config flow +- Introduce MultipartUploadManager: persistent upload metadata on disk, part upload/assembly, restore uploads on startup, listParts/listUploads, abort/cleanup functionality +- Start and stop multipart cleanup task from Smarts3Server lifecycle (startCleanupTask on start, stopCleanupTask on stop) with configurable interval and expiration +- ObjectController: support multipart endpoints (initiate, upload part, complete, abort) and move assembled final object into the object store on completion; set ETag headers and return proper XML responses +- BucketController: support listing in-progress multipart uploads via ?uploads query parameter and return S3-compatible XML +- Persist multipart state to disk and restore on initialization to survive restarts; perform automatic cleanup of expired uploads + ## 2025-11-23 - 5.0.2 - fix(readme) Clarify contribution agreement requirement in README diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 878143c..e8672b6 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smarts3', - version: '5.0.2', + version: '5.1.0', description: 'A Node.js TypeScript package to create a local S3 endpoint for simulating AWS S3 operations using mapped local directories for development and testing purposes.' } diff --git a/ts/classes/multipart-manager.ts b/ts/classes/multipart-manager.ts index 852b9b9..24bcf0e 100644 --- a/ts/classes/multipart-manager.ts +++ b/ts/classes/multipart-manager.ts @@ -23,15 +23,41 @@ export interface IPartInfo { lastModified: Date; } +/** + * Serializable version of upload metadata for disk persistence + */ +interface ISerializableUpload { + uploadId: string; + bucket: string; + key: string; + initiated: string; // ISO date string + metadata: Record; + parts: Array<{ + partNumber: number; + etag: string; + size: number; + lastModified: string; // ISO date string + }>; +} + /** * Manages multipart upload state and storage */ export class MultipartUploadManager { private uploads: Map = new Map(); private uploadDir: string; + private cleanupInterval: NodeJS.Timeout | null = null; + private expirationDays: number; + private cleanupIntervalMinutes: number; - constructor(private rootDir: string) { + constructor( + private rootDir: string, + expirationDays: number = 7, + cleanupIntervalMinutes: number = 60 + ) { this.uploadDir = plugins.path.join(rootDir, '.multipart'); + this.expirationDays = expirationDays; + this.cleanupIntervalMinutes = cleanupIntervalMinutes; } /** @@ -39,6 +65,97 @@ export class MultipartUploadManager { */ public async initialize(): Promise { await plugins.smartfs.directory(this.uploadDir).recursive().create(); + await this.restoreUploadsFromDisk(); + } + + /** + * Save upload metadata to disk for persistence + */ + private async saveUploadMetadata(uploadId: string): Promise { + const upload = this.uploads.get(uploadId); + if (!upload) { + return; + } + + const metadataPath = plugins.path.join(this.uploadDir, uploadId, 'metadata.json'); + + const serializable: ISerializableUpload = { + uploadId: upload.uploadId, + bucket: upload.bucket, + key: upload.key, + initiated: upload.initiated.toISOString(), + metadata: upload.metadata, + parts: Array.from(upload.parts.values()).map(part => ({ + partNumber: part.partNumber, + etag: part.etag, + size: part.size, + lastModified: part.lastModified.toISOString(), + })), + }; + + await plugins.smartfs.file(metadataPath).write(JSON.stringify(serializable, null, 2)); + } + + /** + * Restore uploads from disk on initialization + */ + private async restoreUploadsFromDisk(): Promise { + const uploadDirExists = await plugins.smartfs.directory(this.uploadDir).exists(); + if (!uploadDirExists) { + return; + } + + const entries = await plugins.smartfs.directory(this.uploadDir).includeStats().list(); + + for (const entry of entries) { + if (!entry.isDirectory) { + continue; + } + + const uploadId = entry.name; + const metadataPath = plugins.path.join(this.uploadDir, uploadId, 'metadata.json'); + + // Check if metadata.json exists + const metadataExists = await plugins.smartfs.file(metadataPath).exists(); + if (!metadataExists) { + // Orphaned upload directory - clean it up + console.warn(`Orphaned multipart upload directory found: ${uploadId}, cleaning up`); + await plugins.smartfs.directory(plugins.path.join(this.uploadDir, uploadId)).recursive().delete(); + continue; + } + + try { + // Read and parse metadata + const metadataContent = await plugins.smartfs.file(metadataPath).read(); + const serialized: ISerializableUpload = JSON.parse(metadataContent as string); + + // Restore to memory + const parts = new Map(); + for (const part of serialized.parts) { + parts.set(part.partNumber, { + partNumber: part.partNumber, + etag: part.etag, + size: part.size, + lastModified: new Date(part.lastModified), + }); + } + + this.uploads.set(uploadId, { + uploadId: serialized.uploadId, + bucket: serialized.bucket, + key: serialized.key, + initiated: new Date(serialized.initiated), + parts, + metadata: serialized.metadata, + }); + + console.log(`Restored multipart upload: ${uploadId} (${serialized.bucket}/${serialized.key})`); + } catch (error) { + // Corrupted metadata - clean up + console.error(`Failed to restore multipart upload ${uploadId}:`, error); + await plugins.smartfs.directory(plugins.path.join(this.uploadDir, uploadId)).recursive().delete(); + } + } } /** @@ -71,6 +188,9 @@ export class MultipartUploadManager { const uploadPath = plugins.path.join(this.uploadDir, uploadId); await plugins.smartfs.directory(uploadPath).recursive().create(); + // Persist metadata to disk + await this.saveUploadMetadata(uploadId); + return uploadId; } @@ -116,6 +236,9 @@ export class MultipartUploadManager { upload.parts.set(partNumber, partInfo); + // Persist updated metadata + await this.saveUploadMetadata(uploadId); + return partInfo; } @@ -235,4 +358,73 @@ export class MultipartUploadManager { } return Array.from(upload.parts.values()).sort((a, b) => a.partNumber - b.partNumber); } + + /** + * Start automatic cleanup task for expired uploads + */ + public startCleanupTask(): void { + if (this.cleanupInterval) { + console.warn('Cleanup task is already running'); + return; + } + + // Run cleanup immediately on start + this.performCleanup().catch(err => { + console.error('Failed to perform initial multipart cleanup:', err); + }); + + // Then schedule periodic cleanup + const intervalMs = this.cleanupIntervalMinutes * 60 * 1000; + this.cleanupInterval = setInterval(() => { + this.performCleanup().catch(err => { + console.error('Failed to perform scheduled multipart cleanup:', err); + }); + }, intervalMs); + + console.log(`Multipart cleanup task started (interval: ${this.cleanupIntervalMinutes} minutes, expiration: ${this.expirationDays} days)`); + } + + /** + * Stop automatic cleanup task + */ + public stopCleanupTask(): void { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = null; + console.log('Multipart cleanup task stopped'); + } + } + + /** + * Perform cleanup of expired uploads + */ + private async performCleanup(): Promise { + const now = Date.now(); + const expirationMs = this.expirationDays * 24 * 60 * 60 * 1000; + const expiredUploads: string[] = []; + + // Find expired uploads + for (const [uploadId, upload] of this.uploads.entries()) { + const age = now - upload.initiated.getTime(); + if (age > expirationMs) { + expiredUploads.push(uploadId); + } + } + + if (expiredUploads.length === 0) { + return; + } + + console.log(`Cleaning up ${expiredUploads.length} expired multipart upload(s)`); + + // Delete expired uploads + for (const uploadId of expiredUploads) { + try { + await this.abortUpload(uploadId); + console.log(`Deleted expired multipart upload: ${uploadId}`); + } catch (err) { + console.error(`Failed to delete expired upload ${uploadId}:`, err); + } + } + } } diff --git a/ts/classes/smarts3-server.ts b/ts/classes/smarts3-server.ts index b09cdbb..243e813 100644 --- a/ts/classes/smarts3-server.ts +++ b/ts/classes/smarts3-server.ts @@ -78,11 +78,19 @@ export class Smarts3Server { maxMetadataSize: 2048, requestTimeout: 300000, }, + multipart: { + expirationDays: 7, + cleanupIntervalMinutes: 60, + }, }; this.logger = new Logger(this.config.logging); this.store = new FilesystemStore(this.options.directory); - this.multipart = new MultipartUploadManager(this.options.directory); + this.multipart = new MultipartUploadManager( + this.options.directory, + this.config.multipart.expirationDays, + this.config.multipart.cleanupIntervalMinutes + ); this.router = new S3Router(); this.middlewares = new MiddlewareStack(); @@ -297,6 +305,9 @@ export class Smarts3Server { // Initialize multipart upload manager await this.multipart.initialize(); + // Start multipart cleanup task + this.multipart.startCleanupTask(); + // Clean slate if requested if (this.options.cleanSlate) { await this.store.reset(); @@ -337,6 +348,9 @@ export class Smarts3Server { return; } + // Stop multipart cleanup task + this.multipart.stopCleanupTask(); + await new Promise((resolve, reject) => { this.httpServer!.close((err?: Error) => { if (err) { diff --git a/ts/controllers/bucket.controller.ts b/ts/controllers/bucket.controller.ts index eba6a66..9a4b0dc 100644 --- a/ts/controllers/bucket.controller.ts +++ b/ts/controllers/bucket.controller.ts @@ -54,8 +54,9 @@ export class BucketController { } /** - * GET /:bucket - List objects + * GET /:bucket - List objects or multipart uploads * Supports both V1 and V2 listing (V2 uses list-type=2 query param) + * Multipart uploads listing is triggered by ?uploads query parameter */ public static async listObjects( req: plugins.http.IncomingMessage, @@ -64,6 +65,12 @@ export class BucketController { params: Record ): Promise { const { bucket } = params; + + // Check if this is a ListMultipartUploads request + if (ctx.query.uploads !== undefined) { + return BucketController.listMultipartUploads(req, res, ctx, params); + } + const isV2 = ctx.query['list-type'] === '2'; const result = await ctx.store.listObjects(bucket, { @@ -127,4 +134,47 @@ export class BucketController { }); } } + + /** + * GET /:bucket?uploads - List multipart uploads + */ + private static async listMultipartUploads( + req: plugins.http.IncomingMessage, + res: plugins.http.ServerResponse, + ctx: S3Context, + params: Record + ): Promise { + const { bucket } = params; + + // Get all multipart uploads for this bucket + const uploads = ctx.multipart.listUploads(bucket); + + // Build XML response + await ctx.sendXML({ + ListMultipartUploadsResult: { + '@_xmlns': 'http://s3.amazonaws.com/doc/2006-03-01/', + Bucket: bucket, + KeyMarker: '', + UploadIdMarker: '', + MaxUploads: 1000, + IsTruncated: false, + ...(uploads.length > 0 && { + Upload: uploads.map((upload) => ({ + Key: upload.key, + UploadId: upload.uploadId, + Initiator: { + ID: 'S3RVER', + DisplayName: 'S3RVER', + }, + Owner: { + ID: 'S3RVER', + DisplayName: 'S3RVER', + }, + StorageClass: 'STANDARD', + Initiated: upload.initiated.toISOString(), + })), + }), + }, + }); + } } diff --git a/ts/index.ts b/ts/index.ts index f223184..66f9e82 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -44,6 +44,14 @@ export interface ILimitsConfig { requestTimeout?: number; } +/** + * Multipart upload configuration + */ +export interface IMultipartConfig { + expirationDays?: number; + cleanupIntervalMinutes?: number; +} + /** * Server configuration */ @@ -71,6 +79,7 @@ export interface ISmarts3Config { cors?: ICorsConfig; logging?: ILoggingConfig; limits?: ILimitsConfig; + multipart?: IMultipartConfig; } /** @@ -114,6 +123,10 @@ const DEFAULT_CONFIG: ISmarts3Config = { maxMetadataSize: 2048, requestTimeout: 300000, // 5 minutes }, + multipart: { + expirationDays: 7, + cleanupIntervalMinutes: 60, + }, }; /** @@ -145,6 +158,10 @@ function mergeConfig(userConfig: ISmarts3Config): Required { ...DEFAULT_CONFIG.limits!, ...(userConfig.limits || {}), }, + multipart: { + ...DEFAULT_CONFIG.multipart!, + ...(userConfig.multipart || {}), + }, }; }