Merge pull request 'fix: use overwrite to make metadata files work' (#2) from fix/smartbucket-trash into master

Reviewed-on: #2

changelog.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 2024-11-24 - 3.2.0 - feat(bucket)
+Enhanced SmartBucket with trash management and metadata handling
+
+- Added functionality to move files to a trash directory.
+- Introduced methods to handle file metadata more robustly.
+- Implemented a method to clean all contents from a bucket.
+- Enhanced directory retrieval to handle non-existent directories with options.
+- Improved handling of file paths and metadata within the storage system.
+
 ## 2024-11-18 - 3.1.0 - feat(file)
 Added functionality to retrieve magic bytes from files and detect file types using magic bytes.
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@push.rocks/smartbucket",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@push.rocks/smartbucket",
-      "version": "3.1.0",
+      "version": "3.2.0",
       "license": "UNLICENSED",
       "dependencies": {
         "@push.rocks/smartpath": "^5.0.18",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartbucket",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -15,16 +15,16 @@
     "@git.zone/tsbuild": "^2.1.84",
     "@git.zone/tsrun": "^1.2.49",
     "@git.zone/tstest": "^1.0.90",
-    "@push.rocks/qenv": "^6.0.5",
+    "@push.rocks/qenv": "^6.1.0",
-    "@push.rocks/tapbundle": "^5.3.0"
+    "@push.rocks/tapbundle": "^5.5.3"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.693.0",
+    "@aws-sdk/client-s3": "^3.699.0",
     "@push.rocks/smartmime": "^2.0.4",
     "@push.rocks/smartpath": "^5.0.18",
     "@push.rocks/smartpromise": "^4.0.4",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.2.4",
+    "@push.rocks/smartstream": "^3.2.5",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/smartunique": "^3.0.9",
     "@tsclass/tsclass": "^4.1.2"

test/test.metadata.ts

@@ -0,0 +1,7 @@
+import { tap, expect } from '@push.rocks/tapbundle';
+
+tap.test('test metadata functionality', async () => {
+
+})
+
+tap.start();

test/test.trash.ts

@@ -1,4 +1,5 @@
 import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import { jestExpect } from '@push.rocks/tapbundle/node';
 import { Qenv } from '@push.rocks/qenv';
 
 import * as smartbucket from '../ts/index.js';
@@ -11,18 +12,52 @@ let baseDirectory: smartbucket.Directory;
 
 tap.test('should create a valid smartbucket', async () => {
   testSmartbucket = new smartbucket.SmartBucket({
-    accessKey: await testQenv.getEnvVarOnDemand('S3_KEY'),
+    accessKey: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSKEY'),
-    accessSecret: await testQenv.getEnvVarOnDemand('S3_SECRET'),
+    accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSSECRET'),
-    endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'),
+    endpoint: await testQenv.getEnvVarOnDemandStrict('S3_ENDPOINT'),
   });
   expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
-  myBucket = await testSmartbucket.getBucketByName('testzone');
+  myBucket = await testSmartbucket.getBucketByNameStrict(await testQenv.getEnvVarOnDemandStrict('S3_BUCKET'),);
   expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
-  expect(myBucket.name).toEqual('testzone');
+  expect(myBucket.name).toEqual('test-pushrocks-smartbucket');
 });
 
-tap.test('', async () => {
+tap.test('should clean all contents', async () => {
-  
+  await myBucket.cleanAllContents();
-})
+  expect(await myBucket.fastExists({ path: 'hithere/socool.txt' })).toBeFalse();
+  expect(await myBucket.fastExists({ path: 'trashtest/trashme.txt' })).toBeFalse();
+});
 
-export default tap.start();
+tap.test('should delete a file into the normally', async () => {
+  const path = 'trashtest/trashme.txt';
+  const file = await myBucket.fastPut({
+    path,
+    contents: 'I\'m in the trash test content!',
+  });
+  const fileMetadata = await (await file.getMetaData()).metadataFile.getContents();
+  console.log(fileMetadata.toString());
+  expect(await file.getMetaData().then((meta) => meta.metadataFile.getJsonData())).toEqual({});
+  await file.delete({ mode: 'permanent' });
+  expect((await (await myBucket.getBaseDirectory()).listFiles()).length).toEqual(0);  
+  expect((await (await myBucket.getBaseDirectory()).listDirectories()).length).toEqual(0);  
+});
+
+tap.test('should put a file into the trash', async () => {
+  const path = 'trashtest/trashme.txt';
+  const file = await myBucket.fastPut({
+    path,
+    contents: 'I\'m in the trash test content!',
+  });
+  const fileMetadata = await (await file.getMetaData()).metadataFile.getContents();
+  console.log(fileMetadata.toString());
+  expect(await file.getMetaData().then((meta) => meta.metadataFile.getJsonData())).toEqual({});
+  await file.delete({ mode: 'trash' });
+  jestExpect(await file.getMetaData().then((meta) => meta.metadataFile.getJsonData())).toEqual({
+    custom_recycle: {
+      deletedAt: jestExpect.any(Number),
+      originalPath: "trashtest/trashme.txt",
+    },
+  });
+});
+
+export default tap.start();

test/test.ts

@@ -11,14 +11,20 @@ let baseDirectory: smartbucket.Directory;
 
 tap.test('should create a valid smartbucket', async () => {
   testSmartbucket = new smartbucket.SmartBucket({
-    accessKey: await testQenv.getEnvVarOnDemand('S3_KEY'),
+    accessKey: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSKEY'),
-    accessSecret: await testQenv.getEnvVarOnDemand('S3_SECRET'),
+    accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSSECRET'),
-    endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'),
+    endpoint: await testQenv.getEnvVarOnDemandStrict('S3_ENDPOINT'),
   });
   expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
-  myBucket = await testSmartbucket.getBucketByName('testzone');
+  myBucket = await testSmartbucket.getBucketByNameStrict(await testQenv.getEnvVarOnDemandStrict('S3_BUCKET'),);
   expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
-  expect(myBucket.name).toEqual('testzone');
+  expect(myBucket.name).toEqual('test-pushrocks-smartbucket');
+});
+
+tap.test('should clean all contents', async () => {
+  await myBucket.cleanAllContents();
+  expect(await myBucket.fastExists({ path: 'hithere/socool.txt' })).toBeFalse();
+  expect(await myBucket.fastExists({ path: 'trashtest/trashme.txt' })).toBeFalse();
 });
 
 tap.skip.test('should create testbucket', async () => {
@@ -41,9 +47,12 @@ tap.test('should get data in bucket', async () => {
   const fileString = await myBucket.fastGet({
     path: 'hithere/socool.txt',
   });
-  const fileStringStream = await myBucket.fastGetStream({
+  const fileStringStream = await myBucket.fastGetStream(
-    path: 'hithere/socool.txt',
+    {
-  }, 'nodestream');
+      path: 'hithere/socool.txt',
+    },
+    'nodestream'
+  );
   console.log(fileString);
 });
 
@@ -97,8 +106,9 @@ tap.test('should get base directory', async () => {
 tap.test('should correctly build paths for sub directories', async () => {
   const dir4 = await baseDirectory.getSubDirectoryByName('dir3/dir4');
   expect(dir4).toBeInstanceOf(smartbucket.Directory);
-  const dir4BasePath = dir4.getBasePath();
+  const dir4BasePath = dir4?.getBasePath();
   console.log(dir4BasePath);
+  expect(dir4BasePath).toEqual('dir3/dir4/');
 });
 
 tap.test('clean up directory style tests', async () => {

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartbucket',
-  version: '3.1.0',
+  version: '3.2.0',
   description: 'A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.'
 }

ts/classes.bucket.ts

@@ -52,7 +52,7 @@ export class Bucket {
    * gets the base directory of the bucket
    */
   public async getBaseDirectory(): Promise<Directory> {
-    return new Directory(this, null, '');
+    return new Directory(this, null!, '');
   }
 
   /**
@@ -71,7 +71,9 @@ export class Bucket {
     }
     const checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
     const baseDirectory = await this.getBaseDirectory();
-    return await baseDirectory.getSubDirectoryByName(checkPath);
+    return await baseDirectory.getSubDirectoryByNameStrict(checkPath, {
+      getEmptyDirectory: true,
+    });
   }
 
   // ===============
@@ -331,7 +333,9 @@ export class Bucket {
   }): Promise<void> {
     try {
       const destinationBucket = optionsArg.targetBucket || this;
-      const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath });
+      const exists = await destinationBucket.fastExists({
+        path: optionsArg.destinationPath,
+      });
 
       if (exists && !optionsArg.overwrite) {
         console.error(
@@ -424,8 +428,8 @@ export class Bucket {
       Prefix: checkPath,
       Delimiter: '/',
     });
-    const response = await this.smartbucketRef.s3Client.send(command);
+    const { CommonPrefixes } = await this.smartbucketRef.s3Client.send(command);
-    return response.CommonPrefixes.length > 0;
+    return !!CommonPrefixes && CommonPrefixes.length > 0;
   }
 
   public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
@@ -435,8 +439,8 @@ export class Bucket {
       Prefix: checkPath,
       Delimiter: '/',
     });
-    const response = await this.smartbucketRef.s3Client.send(command);
+    const { Contents } = await this.smartbucketRef.s3Client.send(command);
-    return response.Contents.length > 0;
+    return !!Contents && Contents.length > 0;
   }
 
   public async getMagicBytes(optionsArg: { path: string; length: number }): Promise<Buffer> {
@@ -449,7 +453,7 @@ export class Bucket {
       const response = await this.smartbucketRef.s3Client.send(command);
       const chunks = [];
       const stream = response.Body as any; // SdkStreamMixin includes readable stream
-  
+
       for await (const chunk of stream) {
         chunks.push(chunk);
       }
@@ -462,4 +466,52 @@ export class Bucket {
       throw error;
     }
   }
+
+  public async cleanAllContents(): Promise<void> {
+    try {
+      // Define the command type explicitly
+      const listCommandInput: plugins.s3.ListObjectsV2CommandInput = {
+        Bucket: this.name,
+      };
+  
+      let isTruncated = true;
+      let continuationToken: string | undefined = undefined;
+  
+      while (isTruncated) {
+        // Add the continuation token to the input if present
+        const listCommand = new plugins.s3.ListObjectsV2Command({
+          ...listCommandInput,
+          ContinuationToken: continuationToken,
+        });
+  
+        // Explicitly type the response
+        const response: plugins.s3.ListObjectsV2Output =
+          await this.smartbucketRef.s3Client.send(listCommand);
+        
+        console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
+
+        if (response.Contents && response.Contents.length > 0) {
+          // Delete objects in batches, mapping each item to { Key: string }
+          const deleteCommand = new plugins.s3.DeleteObjectsCommand({
+            Bucket: this.name,
+            Delete: {
+              Objects: response.Contents.map((item) => ({ Key: item.Key! })),
+              Quiet: true,
+            },
+          });
+  
+          await this.smartbucketRef.s3Client.send(deleteCommand);
+        }
+  
+        // Update continuation token and truncation status
+        isTruncated = response.IsTruncated || false;
+        continuationToken = response.NextContinuationToken;
+      }
+  
+      console.log(`All contents in bucket '${this.name}' have been deleted.`);
+    } catch (error) {
+      console.error(`Error cleaning contents of bucket '${this.name}':`, error);
+      throw error;
+    }
+  }
 }

ts/classes.directory.ts

@@ -69,7 +69,7 @@ export class Directory {
     path: string;
     createWithContents?: string | Buffer;
     getFromTrash?: boolean;
-  }): Promise<File> {
+  }): Promise<File | null> {
     const pathDescriptor = {
       directory: this,
       path: optionsArg.path,
@@ -98,6 +98,19 @@ export class Directory {
     });
   }
 
+  /**
+   * gets a file strictly
+   * @param args 
+   * @returns 
+   */
+  public async getFileStrict(...args: Parameters<Directory['getFile']>) {
+    const file = await this.getFile(...args);
+    if (!file) {
+      throw new Error(`File not found at path '${args[0].path}'`);
+    }
+    return file;
+  }
+
   /**
    * lists all files
    */
@@ -110,7 +123,7 @@ export class Directory {
     const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
     const fileArray: File[] = [];
 
-    response.Contents.forEach((item) => {
+    response.Contents?.forEach((item) => {
       if (item.Key && !item.Key.endsWith('/')) {
         const subtractedPath = item.Key.replace(this.getBasePath(), '');
         if (!subtractedPath.includes('/')) {
@@ -178,23 +191,53 @@ export class Directory {
   /**
    * gets a sub directory by name
    */
-  public async getSubDirectoryByName(dirNameArg: string): Promise<Directory> {
+  public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
-    const dirNameArray = dirNameArg.split('/');
+    getEmptyDirectory?: boolean;
+    createWithInitializerFile?: boolean;
+  } = {}): Promise<Directory | null> {
+    const dirNameArray = dirNameArg.split('/').filter(str => str.trim() !== "");
 
-    const getDirectory = async (directoryArg: Directory, dirNameToSearch: string) => {
+    optionsArg = {
-      const directories = await directoryArg.listDirectories();
+      getEmptyDirectory: false,
-      return directories.find((directory) => {
+      createWithInitializerFile: false,
-        return directory.name === dirNameToSearch;
+      ...optionsArg,
-      });
-    };
-
-    let wantedDirectory: Directory;
-    for (const dirNameToSearch of dirNameArray) {
-      const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
-      wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch);
     }
 
-    return wantedDirectory;
+
+    const getDirectory = async (directoryArg: Directory, dirNameToSearch: string, isFinalDirectory: boolean) => {
+      const directories = await directoryArg.listDirectories();
+      let returnDirectory = directories.find((directory) => {
+        return directory.name === dirNameToSearch;
+      });
+      if (returnDirectory) {
+        return returnDirectory;
+      }
+      if (optionsArg.getEmptyDirectory || optionsArg.createWithInitializerFile) {
+        returnDirectory = new Directory(this.bucketRef, this, dirNameToSearch);
+      }
+      if (isFinalDirectory && optionsArg.createWithInitializerFile) {
+        returnDirectory?.createEmptyFile('00init.txt');
+      }
+      return returnDirectory || null;
+    };
+
+    let wantedDirectory: Directory | null = null;
+    let counter = 0; 
+    for (const dirNameToSearch of dirNameArray) {
+      counter++;
+      const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
+      wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch, counter === dirNameArray.length);
+    }
+
+    return wantedDirectory || null;
+  }
+
+  public async getSubDirectoryByNameStrict(...args: Parameters<Directory['getSubDirectoryByName']>) {
+    const directory = await this.getSubDirectoryByName(...args);
+    if (!directory) {
+      throw new Error(`Directory not found at path '${args[0]}'`);
+    }
+    return directory;
   }
 
   /**

ts/classes.file.ts

@@ -92,24 +92,23 @@ export class File {
   /**
    * deletes this file
    */
-  public async delete(optionsArg?: {
+  public async delete(optionsArg?: { mode: 'trash' | 'permanent' }) {
-    mode: 'trash' | 'permanent';
-  }) {
-
     optionsArg = {
-      ... {
+      ...{
         mode: 'permanent',
       },
       ...optionsArg,
-    }
+    };
 
     if (optionsArg.mode === 'permanent') {
       await this.parentDirectoryRef.bucketRef.fastRemove({
         path: this.getBasePath(),
       });
       if (!this.name.endsWith('.metadata')) {
-        const metadata = await this.getMetaData();
+        if (await this.hasMetaData()) {
-        await metadata.metadataFile.delete(optionsArg);
+          const metadata = await this.getMetaData();
+          await metadata.metadataFile.delete(optionsArg);
+        }
       }
     } else if (optionsArg.mode === 'trash') {
       const metadata = await this.getMetaData();
@@ -121,12 +120,13 @@ export class File {
         },
       });
       const trash = await this.parentDirectoryRef.bucketRef.getTrash();
+      const trashDir = await trash.getTrashDir();
       await this.move({
-        directory: await trash.getTrashDir(),
+        directory: trashDir,
         path: await trash.getTrashKeyByOriginalBasePath(this.getBasePath()),
       });
     }
-    
+
     await this.parentDirectoryRef.listFiles();
   }
 
@@ -169,16 +169,19 @@ export class File {
       await this.parentDirectoryRef.bucketRef.fastPutStream({
         path: this.getBasePath(),
         readableStream: optionsArg.contents,
+        overwrite: true,
       });
     } else if (Buffer.isBuffer(optionsArg.contents)) {
       await this.parentDirectoryRef.bucketRef.fastPut({
         path: this.getBasePath(),
         contents: optionsArg.contents,
+        overwrite: true,
       });
     } else if (typeof optionsArg.contents === 'string') {
       await this.parentDirectoryRef.bucketRef.fastPut({
         path: this.getBasePath(),
         contents: Buffer.from(optionsArg.contents, optionsArg.encoding),
+        overwrite: true,
       });
     }
   }
@@ -187,23 +190,49 @@ export class File {
    * moves the file to another directory
    */
   public async move(pathDescriptorArg: interfaces.IPathDecriptor) {
-    let moveToPath = '';
+    let moveToPath: string = '';
     const isDirectory = await this.parentDirectoryRef.bucketRef.isDirectory(pathDescriptorArg);
     if (isDirectory) {
       moveToPath = await helpers.reducePathDescriptorToPath({
         ...pathDescriptorArg,
         path: plugins.path.join(pathDescriptorArg.path!, this.name),
       });
+    } else {
+      moveToPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
     }
     // lets move the file
     await this.parentDirectoryRef.bucketRef.fastMove({
       sourcePath: this.getBasePath(),
       destinationPath: moveToPath,
+      overwrite: true,
     });
-
     // lets move the metadatafile
-    const metadata = await this.getMetaData();
+    if (!this.name.endsWith('.metadata')) {
-    await metadata.metadataFile.move(pathDescriptorArg);
+      const metadata = await this.getMetaData();
+      await this.parentDirectoryRef.bucketRef.fastMove({
+        sourcePath: metadata.metadataFile.getBasePath(),
+        destinationPath: moveToPath + '.metadata',
+        overwrite: true,
+      });
+    }
+
+    // lets update references of this
+    const baseDirectory = await this.parentDirectoryRef.bucketRef.getBaseDirectory();
+    this.parentDirectoryRef = await baseDirectory.getSubDirectoryByNameStrict(
+      pathDescriptorArg.directory?.getBasePath()!
+    );
+    this.name = pathDescriptorArg.path!;
+  }
+
+  public async hasMetaData(): Promise<boolean> {
+    if (!this.name.endsWith('.metadata')) {
+      const hasMetadataBool = MetaData.hasMetaData({
+        file: this,
+      });
+      return hasMetadataBool;
+    } else {
+      return false;
+    }
   }
 
   /**
@@ -238,7 +267,7 @@ export class File {
   public async getMagicBytes(optionsArg: { length: number }): Promise<Buffer> {
     return this.parentDirectoryRef.bucketRef.getMagicBytes({
       path: this.getBasePath(),
-      length: optionsArg.length
+      length: optionsArg.length,
-    })
+    });
   }
 }

ts/classes.metadata.ts

@@ -3,13 +3,21 @@ import * as plugins from './plugins.js';
 import { File } from './classes.file.js';
 
 export class MetaData {
+  public static async hasMetaData(optionsArg: { file: File }) {
+    // lets find the existing metadata file
+    const existingFile = await optionsArg.file.parentDirectoryRef.getFile({
+      path: optionsArg.file.name + '.metadata',
+    });
+    return !!existingFile;
+  }
+
   // static
   public static async createForFile(optionsArg: { file: File }) {
     const metaData = new MetaData();
     metaData.fileRef = optionsArg.file;
 
     // lets find the existing metadata file
-    metaData.metadataFile = await metaData.fileRef.parentDirectoryRef.getFile({
+    metaData.metadataFile = await metaData.fileRef.parentDirectoryRef.getFileStrict({
       path: metaData.fileRef.name + '.metadata',
       createWithContents: '{}',
     });

ts/classes.smartbucket.ts

@@ -41,7 +41,15 @@ export class SmartBucket {
     await Bucket.removeBucketByName(this, bucketName);
   }
 
-  public async getBucketByName(bucketName: string) {
+  public async getBucketByName(bucketNameArg: string) {
-    return Bucket.getBucketByName(this, bucketName);
+    return Bucket.getBucketByName(this, bucketNameArg);
+  }
+
+  public async getBucketByNameStrict(...args: Parameters<SmartBucket['getBucketByName']>) {
+    const bucket = await this.getBucketByName(...args);
+    if (!bucket) {
+      throw new Error(`Bucket ${args[0]} does not exist.`);
+    }
+    return bucket;
   }
 }

ts/classes.trash.ts

@@ -21,7 +21,7 @@ export class Trash {
     const trashDir = await this.getTrashDir();
     const originalPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
     const trashKey = await this.getTrashKeyByOriginalBasePath(originalPath);
-    return trashDir.getFile({ path: trashKey });
+    return trashDir.getFileStrict({ path: trashKey });
   }
 
   public async getTrashKeyByOriginalBasePath (originalPath: string): Promise<string> {