From 1f4b7319d380e61a38d67d396ebf0df6f846ed18 Mon Sep 17 00:00:00 2001 From: Juergen Kunz Date: Thu, 20 Nov 2025 13:58:02 +0000 Subject: [PATCH] feat(core): Add S3 endpoint normalization, directory pagination, improved metadata checks, trash support, and related tests --- changelog.md | 12 ++++++ test/test.local.node+deno.ts | 76 ++++++++++++++++++++++++++++++++++++ ts/00_commitinfo_data.ts | 2 +- ts/classes.directory.ts | 52 ++++++++++++++++-------- ts/classes.metadata.ts | 22 ++++++++--- ts/index.ts | 2 + 6 files changed, 144 insertions(+), 22 deletions(-) create mode 100644 test/test.local.node+deno.ts diff --git a/changelog.md b/changelog.md index 56b3b2c..5b32946 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,17 @@ # Changelog +## 2025-11-20 - 4.1.0 - feat(core) +Add S3 endpoint normalization, directory pagination, improved metadata checks, trash support, and related tests + +- Add normalizeS3Descriptor helper to sanitize and normalize various S3 endpoint formats and emit warnings for mismatches (helpers.ts). +- Use normalized endpoint and credentials when constructing S3 client in SmartBucket (classes.smartbucket.ts). +- Implement paginated listing helper listObjectsV2AllPages in Directory and use it for listFiles and listDirectories to aggregate Contents and CommonPrefixes across pages (classes.directory.ts). +- Improve MetaData.hasMetaData to catch NotFound errors and return false instead of throwing (classes.metadata.ts). +- Export metadata and trash modules from index (ts/index.ts) and add a Trash class with utilities for trashed files and key encoding (classes.trash.ts). +- Enhance Bucket operations: fastCopy now preserves or replaces native metadata correctly, cleanAllContents supports paginated deletion, and improved fastExists error handling (classes.bucket.ts). +- Fix Directory.getSubDirectoryByName to construct new Directory instances with the correct parent directory reference. +- Add tests covering metadata absence and pagination behavior (test/test.local.node+deno.ts). + ## 2025-11-20 - 4.0.1 - fix(plugins) Use explicit node: imports for native path and stream modules in ts/plugins.ts diff --git a/test/test.local.node+deno.ts b/test/test.local.node+deno.ts new file mode 100644 index 0000000..7f45530 --- /dev/null +++ b/test/test.local.node+deno.ts @@ -0,0 +1,76 @@ +import { expect, tap } from '@git.zone/tstest/tapbundle'; + +import * as plugins from '../ts/plugins.js'; +import * as smartbucket from '../ts/index.js'; + +class FakeS3Client { + private callIndex = 0; + + constructor(private readonly pages: Array>) {} + + public async send(_command: any) { + const page = this.pages[this.callIndex] || { Contents: [], CommonPrefixes: [], IsTruncated: false }; + this.callIndex += 1; + return page; + } +} + +tap.test('MetaData.hasMetaData should return false when metadata file does not exist', async () => { + const fakeFile = { + name: 'file.txt', + parentDirectoryRef: { + async getFile() { + throw new Error(`File not found at path 'file.txt.metadata'`); + }, + }, + } as unknown as smartbucket.File; + + const hasMetaData = await smartbucket.MetaData.hasMetaData({ file: fakeFile }); + expect(hasMetaData).toBeFalse(); +}); + +tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => { + const fakeSmartbucket = { s3Client: new FakeS3Client([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket; + const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket'); + const baseDirectory = new smartbucket.Directory(bucket, null as any, ''); + + const nestedDirectory = await baseDirectory.getSubDirectoryByName('level1/level2', { getEmptyDirectory: true }); + + expect(nestedDirectory.name).toEqual('level2'); + expect(nestedDirectory.parentDirectoryRef.name).toEqual('level1'); + expect(nestedDirectory.getBasePath()).toEqual('level1/level2/'); +}); + +tap.test('listFiles should aggregate results across paginated ListObjectsV2 responses', async () => { + const firstPage = { + Contents: Array.from({ length: 1000 }, (_, index) => ({ Key: `file-${index}` })), + IsTruncated: true, + NextContinuationToken: 'token-1', + }; + const secondPage = { + Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })), + IsTruncated: false, + }; + const fakeSmartbucket = { s3Client: new FakeS3Client([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket; + const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket'); + const baseDirectory = new smartbucket.Directory(bucket, null as any, ''); + + const files = await baseDirectory.listFiles(); + expect(files.length).toEqual(1200); +}); + +tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => { + const fakeSmartbucket = { + s3Client: new FakeS3Client([ + { CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' }, + { CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false }, + ]), + } as unknown as smartbucket.SmartBucket; + const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket'); + const baseDirectory = new smartbucket.Directory(bucket, null as any, ''); + + const directories = await baseDirectory.listDirectories(); + expect(directories.map((d) => d.name)).toEqual(['dirA', 'dirB']); +}); + +export default tap.start(); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 6403697..b49a600 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartbucket', - version: '4.0.1', + version: '4.1.0', description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.' } diff --git a/ts/classes.directory.ts b/ts/classes.directory.ts index 4e274d2..85915b6 100644 --- a/ts/classes.directory.ts +++ b/ts/classes.directory.ts @@ -120,19 +120,44 @@ export class Directory { return directories.some(dir => dir.name === dirNameArg); } + /** + * Collects all ListObjectsV2 pages for a prefix. + */ + private async listObjectsV2AllPages(prefix: string, delimiter?: string) { + const allContents: plugins.s3._Object[] = []; + const allCommonPrefixes: plugins.s3.CommonPrefix[] = []; + let continuationToken: string | undefined; + + do { + const command = new plugins.s3.ListObjectsV2Command({ + Bucket: this.bucketRef.name, + Prefix: prefix, + Delimiter: delimiter, + ContinuationToken: continuationToken, + }); + const response = await this.bucketRef.smartbucketRef.s3Client.send(command); + + if (response.Contents) { + allContents.push(...response.Contents); + } + if (response.CommonPrefixes) { + allCommonPrefixes.push(...response.CommonPrefixes); + } + + continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined; + } while (continuationToken); + + return { contents: allContents, commonPrefixes: allCommonPrefixes }; + } + /** * lists all files */ public async listFiles(): Promise { - const command = new plugins.s3.ListObjectsV2Command({ - Bucket: this.bucketRef.name, - Prefix: this.getBasePath(), - Delimiter: '/', - }); - const response = await this.bucketRef.smartbucketRef.s3Client.send(command); + const { contents } = await this.listObjectsV2AllPages(this.getBasePath(), '/'); const fileArray: File[] = []; - response.Contents?.forEach((item) => { + contents.forEach((item) => { if (item.Key && !item.Key.endsWith('/')) { const subtractedPath = item.Key.replace(this.getBasePath(), ''); if (!subtractedPath.includes('/')) { @@ -154,16 +179,11 @@ export class Directory { */ public async listDirectories(): Promise { try { - const command = new plugins.s3.ListObjectsV2Command({ - Bucket: this.bucketRef.name, - Prefix: this.getBasePath(), - Delimiter: '/', - }); - const response = await this.bucketRef.smartbucketRef.s3Client.send(command); + const { commonPrefixes } = await this.listObjectsV2AllPages(this.getBasePath(), '/'); const directoryArray: Directory[] = []; - if (response.CommonPrefixes) { - response.CommonPrefixes.forEach((item) => { + if (commonPrefixes) { + commonPrefixes.forEach((item) => { if (item.Prefix) { const subtractedPath = item.Prefix.replace(this.getBasePath(), ''); if (subtractedPath.endsWith('/')) { @@ -235,7 +255,7 @@ export class Directory { return returnDirectory; } if (optionsArg.getEmptyDirectory || optionsArg.createWithInitializerFile) { - returnDirectory = new Directory(this.bucketRef, this, dirNameToSearch); + returnDirectory = new Directory(this.bucketRef, directoryArg, dirNameToSearch); } if (isFinalDirectory && optionsArg.createWithInitializerFile) { returnDirectory?.createEmptyFile('00init.txt'); diff --git a/ts/classes.metadata.ts b/ts/classes.metadata.ts index 0a80e55..1e62422 100644 --- a/ts/classes.metadata.ts +++ b/ts/classes.metadata.ts @@ -4,11 +4,23 @@ import { File } from './classes.file.js'; export class MetaData { public static async hasMetaData(optionsArg: { file: File }) { - // lets find the existing metadata file - const existingFile = await optionsArg.file.parentDirectoryRef.getFile({ - path: optionsArg.file.name + '.metadata', - }); - return !!existingFile; + // try finding the existing metadata file; return false if it doesn't exist + try { + const existingFile = await optionsArg.file.parentDirectoryRef.getFile({ + path: optionsArg.file.name + '.metadata', + }); + return !!existingFile; + } catch (error: any) { + const message = error?.message || ''; + const isNotFound = + message.includes('File not found') || + error?.name === 'NotFound' || + error?.$metadata?.httpStatusCode === 404; + if (isNotFound) { + return false; + } + throw error; + } } // static diff --git a/ts/index.ts b/ts/index.ts index 1aea0bc..5689962 100644 --- a/ts/index.ts +++ b/ts/index.ts @@ -2,3 +2,5 @@ export * from './classes.smartbucket.js'; export * from './classes.bucket.js'; export * from './classes.directory.js'; export * from './classes.file.js'; +export * from './classes.metadata.js'; +export * from './classes.trash.js';