3.1.0

changelog.md

@@ -0,0 +1,86 @@
+# Changelog
+
+## 2024-11-18 - 3.1.0 - feat(file)
+Added functionality to retrieve magic bytes from files and detect file types using magic bytes.
+
+- Introduced method `getMagicBytes` in `File` and `Bucket` classes to retrieve a specific number of bytes from a file.
+- Enhanced file type detection by utilizing magic bytes in `MetaData` class.
+- Updated dependencies for better performance and compatibility.
+
+## 2024-11-18 - 3.0.24 - fix(metadata)
+Fix metadata handling to address type assertion and data retrieval.
+
+- Fixed type assertion issues in `MetaData` class properties with type non-null assertions.
+- Corrected the handling of JSON data retrieval in `MetaData.storeCustomMetaData` function.
+
+## 2024-10-16 - 3.0.23 - fix(dependencies)
+Update package dependencies for improved functionality and security.
+
+- Updated @aws-sdk/client-s3 to version ^3.670.0 for enhanced S3 client capabilities.
+- Updated @push.rocks/smartstream to version ^3.2.4.
+- Updated the dev dependency @push.rocks/tapbundle to version ^5.3.0.
+
+## 2024-07-28 - 3.0.22 - fix(dependencies)
+Update dependencies and improve bucket retrieval logging
+
+- Updated @aws-sdk/client-s3 to ^3.620.0
+- Updated @git.zone/tsbuild to ^2.1.84
+- Updated @git.zone/tsrun to ^1.2.49
+- Updated @push.rocks/smartpromise to ^4.0.4
+- Updated @tsclass/tsclass to ^4.1.2
+- Added a log for when a bucket is not found by name in getBucketByName method
+
+## 2024-07-04 - 3.0.21 - fix(test)
+Update endpoint configuration in tests to use environment variable
+
+- Modified `qenv.yml` to include `S3_ENDPOINT` as a required environment variable.
+- Updated test files to fetch `S3_ENDPOINT` from environment instead of hardcoding.
+
+## 2024-06-19 - 3.0.20 - Fix and Stability Updates
+Improved overall stability and consistency.
+
+## 2024-06-18 - 3.0.18 - Delete Functions Consistency
+Ensured more consistency between delete methods and trash behavior.
+
+## 2024-06-17 - 3.0.17 to 3.0.16 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-06-11 - 3.0.15 to 3.0.14 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-06-10 - 3.0.13 - Trash Feature Completion
+Finished work on trash feature.
+
+## 2024-06-09 - 3.0.12 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-06-08 - 3.0.11 to 3.0.10 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-06-03 - 3.0.10 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-05-29 - 3.0.9 - Update Description
+Updated project description.
+
+## 2024-05-27 - 3.0.8 to 3.0.6 - Pathing and Core Updates
+Routine updates and fixes performed.
+  - S3 paths' pathing differences now correctly handled with a reducePath method.
+
+## 2024-05-21 - 3.0.5 to 3.0.4 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-05-17 - 3.0.3 to 3.0.2 - Fix and Update
+Routine updates and fixes performed.
+
+## 2024-05-17 - 3.0.0 - Major Release
+Introduced breaking changes in core and significant improvements.
+
+## 2024-05-05 - 2.0.5 - Breaking Changes
+Introduced breaking changes in core functionality.
+
+## 2024-04-14 - 2.0.4 - TSConfig Update
+Updated TypeScript configuration.
+
+## 2024-01-01 - 2.0.2 - Organization Scheme Update
+Switched to the new organizational scheme.

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@push.rocks/smartbucket",
-  "version": "3.0.16",
+  "version": "3.1.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@push.rocks/smartbucket",
-      "version": "3.0.16",
+      "version": "3.1.0",
       "license": "UNLICENSED",
       "dependencies": {
         "@push.rocks/smartpath": "^5.0.18",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@push.rocks/smartbucket",
-  "version": "3.0.16",
+  "version": "3.1.0",
   "description": "A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.",
   "main": "dist_ts/index.js",
   "typings": "dist_ts/index.d.ts",
@@ -12,22 +12,22 @@
     "build": "(tsbuild --web --allowimplicitany)"
   },
   "devDependencies": {
-    "@git.zone/tsbuild": "^2.1.80",
-    "@git.zone/tsrun": "^1.2.46",
+    "@git.zone/tsbuild": "^2.1.84",
+    "@git.zone/tsrun": "^1.2.49",
     "@git.zone/tstest": "^1.0.90",
     "@push.rocks/qenv": "^6.0.5",
-    "@push.rocks/tapbundle": "^5.0.23"
+    "@push.rocks/tapbundle": "^5.3.0"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.598.0",
-    "@push.rocks/smartmime": "^2.0.2",
+    "@aws-sdk/client-s3": "^3.693.0",
+    "@push.rocks/smartmime": "^2.0.4",
     "@push.rocks/smartpath": "^5.0.18",
-    "@push.rocks/smartpromise": "^4.0.3",
+    "@push.rocks/smartpromise": "^4.0.4",
     "@push.rocks/smartrx": "^3.0.7",
-    "@push.rocks/smartstream": "^3.0.44",
+    "@push.rocks/smartstream": "^3.2.4",
     "@push.rocks/smartstring": "^4.0.15",
     "@push.rocks/smartunique": "^3.0.9",
-    "@tsclass/tsclass": "^4.0.59"
+    "@tsclass/tsclass": "^4.1.2"
   },
   "private": false,
   "files": [

qenv.yml

@@ -1,3 +1,4 @@
 required:
   - S3_KEY
-  - S3_SECRET
+  - S3_SECRET
+  - S3_ENDPOINT

test/test.trash.ts

@@ -0,0 +1,28 @@
+import { expect, expectAsync, tap } from '@push.rocks/tapbundle';
+import { Qenv } from '@push.rocks/qenv';
+
+import * as smartbucket from '../ts/index.js';
+
+const testQenv = new Qenv('./', './.nogit/');
+
+let testSmartbucket: smartbucket.SmartBucket;
+let myBucket: smartbucket.Bucket;
+let baseDirectory: smartbucket.Directory;
+
+tap.test('should create a valid smartbucket', async () => {
+  testSmartbucket = new smartbucket.SmartBucket({
+    accessKey: await testQenv.getEnvVarOnDemand('S3_KEY'),
+    accessSecret: await testQenv.getEnvVarOnDemand('S3_SECRET'),
+    endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'),
+  });
+  expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
+  myBucket = await testSmartbucket.getBucketByName('testzone');
+  expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
+  expect(myBucket.name).toEqual('testzone');
+});
+
+tap.test('', async () => {
+  
+})
+
+export default tap.start();

test/test.ts

@@ -13,24 +13,22 @@ tap.test('should create a valid smartbucket', async () => {
   testSmartbucket = new smartbucket.SmartBucket({
     accessKey: await testQenv.getEnvVarOnDemand('S3_KEY'),
     accessSecret: await testQenv.getEnvVarOnDemand('S3_SECRET'),
-    endpoint: 's3.eu-central-1.wasabisys.com',
+    endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'),
   });
-});
-
-tap.skip.test('should create testbucket', async () => {
-  // await testSmartbucket.createBucket('testzone');
-});
-
-tap.skip.test('should remove testbucket', async () => {
-  // await testSmartbucket.removeBucket('testzone');
-});
-
-tap.test('should get a bucket', async () => {
+  expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
   myBucket = await testSmartbucket.getBucketByName('testzone');
   expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
   expect(myBucket.name).toEqual('testzone');
 });
 
+tap.skip.test('should create testbucket', async () => {
+  // await testSmartbucket.createBucket('testzone2');
+});
+
+tap.skip.test('should remove testbucket', async () => {
+  // await testSmartbucket.removeBucket('testzone2');
+});
+
 // Fast operations
 tap.test('should store data in bucket fast', async () => {
   await myBucket.fastPut({

ts/00_commitinfo_data.ts

@@ -1,8 +1,8 @@
 /**
- * autocreated commitinfo by @pushrocks/commitinfo
+ * autocreated commitinfo by @push.rocks/commitinfo
  */
 export const commitinfo = {
   name: '@push.rocks/smartbucket',
-  version: '3.0.16',
+  version: '3.1.0',
   description: 'A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.'
 }

ts/classes.bucket.ts

@@ -24,6 +24,7 @@ export class Bucket {
       console.log(`Taking this as base for new Bucket instance`);
       return new this(smartbucketRef, bucketNameArg);
     } else {
+      console.log(`did not find bucket by name: ${bucketNameArg}`);
       return null;
     }
   }
@@ -231,6 +232,7 @@ export class Bucket {
     if (typeArg === 'webstream') {
       return (await duplexStream.getWebStreams()).readable;
     }
+    throw new Error('unknown typeArg');
   }
 
   /**
@@ -436,4 +438,28 @@ export class Bucket {
     const response = await this.smartbucketRef.s3Client.send(command);
     return response.Contents.length > 0;
   }
+
+  public async getMagicBytes(optionsArg: { path: string; length: number }): Promise<Buffer> {
+    try {
+      const command = new plugins.s3.GetObjectCommand({
+        Bucket: this.name,
+        Key: optionsArg.path,
+        Range: `bytes=0-${optionsArg.length - 1}`,
+      });
+      const response = await this.smartbucketRef.s3Client.send(command);
+      const chunks = [];
+      const stream = response.Body as any; // SdkStreamMixin includes readable stream
+  
+      for await (const chunk of stream) {
+        chunks.push(chunk);
+      }
+      return Buffer.concat(chunks);
+    } catch (error) {
+      console.error(
+        `Error retrieving magic bytes from object at path '${optionsArg.path}' in bucket '${this.name}':`,
+        error
+      );
+      throw error;
+    }
+  }
 }

ts/classes.directory.ts

@@ -66,13 +66,13 @@ export class Directory {
    * gets a file by name
    */
   public async getFile(optionsArg: {
-    name: string;
+    path: string;
     createWithContents?: string | Buffer;
     getFromTrash?: boolean;
   }): Promise<File> {
     const pathDescriptor = {
       directory: this,
-      path: optionsArg.name,
+      path: optionsArg.path,
     };
     const exists = await this.bucketRef.fastExists({
       path: await helpers.reducePathDescriptorToPath(pathDescriptor),
@@ -88,13 +88,13 @@ export class Directory {
     if (!exists && optionsArg.createWithContents) {
       await File.create({
         directory: this,
-        name: optionsArg.name,
+        name: optionsArg.path,
         contents: optionsArg.createWithContents,
       });
     }
     return new File({
       directoryRefArg: this,
-      fileName: optionsArg.name,
+      fileName: optionsArg.path,
     });
   }
 
@@ -283,19 +283,30 @@ export class Directory {
 
   /**
    * removes a file within the directory
+   * uses file class to make sure effects for metadata etc. are handled correctly
    * @param optionsArg
    */
-  public async fastRemove(optionsArg: { path: string }) {
-    const path = plugins.path.join(this.getBasePath(), optionsArg.path);
-    await this.bucketRef.fastRemove({
-      path,
+  public async fastRemove(optionsArg: {
+    path: string
+    /**
+     * wether the file should be placed into trash. Default is false.
+     */
+    mode?: 'permanent' | 'trash';
+  }) {
+    const file = await this.getFile({
+      path: optionsArg.path,
+    });
+    await file.delete({
+      mode: optionsArg.mode ? optionsArg.mode : 'permanent',
     });
   }
 
   /**
    * deletes the directory with all its contents
    */
-  public async delete() {
+  public async delete(optionsArg: {
+    mode?: 'permanent' | 'trash';
+  }) {
     const deleteDirectory = async (directoryArg: Directory) => {
       const childDirectories = await directoryArg.listDirectories();
       if (childDirectories.length === 0) {
@@ -307,9 +318,9 @@ export class Directory {
       }
       const files = await directoryArg.listFiles();
       for (const file of files) {
-        await directoryArg.fastRemove({
-          path: file.name,
-        });
+        await file.delete({
+          mode: optionsArg.mode ? optionsArg.mode : 'permanent',
+        })
       }
     };
     await deleteDirectory(this);

ts/classes.file.ts

@@ -50,6 +50,10 @@ export class File {
   public parentDirectoryRef: Directory;
   public name: string;
 
+  /**
+   * get the full path to the file
+   * @returns the full path to the file
+   */
   public getBasePath(): string {
     return plugins.path.join(this.parentDirectoryRef.getBasePath(), this.name);
   }
@@ -188,7 +192,7 @@ export class File {
     if (isDirectory) {
       moveToPath = await helpers.reducePathDescriptorToPath({
         ...pathDescriptorArg,
-        path: plugins.path.join(pathDescriptorArg.path, this.name),
+        path: plugins.path.join(pathDescriptorArg.path!, this.name),
       });
     }
     // lets move the file
@@ -230,4 +234,11 @@ export class File {
       contents: JSON.stringify(dataArg),
     });
   }
+
+  public async getMagicBytes(optionsArg: { length: number }): Promise<Buffer> {
+    return this.parentDirectoryRef.bucketRef.getMagicBytes({
+      path: this.getBasePath(),
+      length: optionsArg.length
+    })
+  }
 }

ts/classes.metadata.ts

@@ -10,7 +10,7 @@ export class MetaData {
 
     // lets find the existing metadata file
     metaData.metadataFile = await metaData.fileRef.parentDirectoryRef.getFile({
-      name: metaData.fileRef.name + '.metadata',
+      path: metaData.fileRef.name + '.metadata',
       createWithContents: '{}',
     });
 
@@ -21,20 +21,34 @@ export class MetaData {
   /**
    * the file that contains the metadata
    */
-  metadataFile: File;
+  metadataFile!: File;
 
   /**
    * the file that the metadata is for
    */
-  fileRef: File;
+  fileRef!: File;
 
   public async getFileType(optionsArg?: {
     useFileExtension?: boolean;
     useMagicBytes?: boolean;
-  }): Promise<string> {
-    if ((optionsArg && optionsArg.useFileExtension) || optionsArg.useFileExtension === undefined) {
-      return plugins.path.extname(this.fileRef.name);
+  }): Promise<plugins.smartmime.IFileTypeResult | undefined> {
+    if ((optionsArg && optionsArg.useFileExtension) || !optionsArg) {
+      const fileType = await plugins.smartmime.detectMimeType({
+        path: this.fileRef.name,
+      });
+
+      return fileType;
     }
+    if (optionsArg && optionsArg.useMagicBytes) {
+      const fileType = await plugins.smartmime.detectMimeType({
+        buffer: await this.fileRef.getMagicBytes({
+          length: 100,
+        })
+      });
+
+      return fileType;
+    }
+    throw new Error('optionsArg.useFileExtension and optionsArg.useMagicBytes cannot both be false');
   }
 
   /**
@@ -44,13 +58,13 @@ export class MetaData {
     const stat = await this.fileRef.parentDirectoryRef.bucketRef.fastStat({
       path: this.fileRef.getBasePath(),
     });
-    return stat.ContentLength;
+    return stat.ContentLength!;
   }
 
   private prefixCustomMetaData = 'custom_';
 
   public async storeCustomMetaData<T = any>(optionsArg: { key: string; value: T }) {
-    const data = await this.metadataFile.getContentsAsString();
+    const data = await this.metadataFile.getJsonData();
     data[this.prefixCustomMetaData + optionsArg.key] = optionsArg.value;
     await this.metadataFile.writeJsonData(data);
   }

ts/classes.smartbucket.ts

@@ -16,16 +16,17 @@ export class SmartBucket {
    */
   constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
     this.config = configArg;
-    const endpoint = this.config.endpoint.startsWith('http://') || this.config.endpoint.startsWith('https://')
-      ? this.config.endpoint
-      : `https://${this.config.endpoint}`;
+
+    const protocol = configArg.useSsl === false ? 'http' : 'https';
+    const port = configArg.port ? `:${configArg.port}` : '';
+    const endpoint = `${protocol}://${configArg.endpoint}${port}`;
 
     this.s3Client = new plugins.s3.S3Client({
       endpoint,
-      region: this.config.region || 'us-east-1',
+      region: configArg.region || 'us-east-1',
       credentials: {
-        accessKeyId: this.config.accessKey,
-        secretAccessKey: this.config.accessSecret,
+        accessKeyId: configArg.accessKey,
+        secretAccessKey: configArg.accessSecret,
       },
       forcePathStyle: true, // Necessary for S3-compatible storage like MinIO or Wasabi
     });
@@ -43,4 +44,4 @@ export class SmartBucket {
   public async getBucketByName(bucketName: string) {
     return Bucket.getBucketByName(this, bucketName);
   }
-}
+}

ts/classes.trash.ts

@@ -21,7 +21,7 @@ export class Trash {
     const trashDir = await this.getTrashDir();
     const originalPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
     const trashKey = await this.getTrashKeyByOriginalBasePath(originalPath);
-    return trashDir.getFile({ name: trashKey });
+    return trashDir.getFile({ path: trashKey });
   }
 
   public async getTrashKeyByOriginalBasePath (originalPath: string): Promise<string> {

tsconfig.json

@@ -6,7 +6,8 @@
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "esModuleInterop": true,
-    "verbatimModuleSyntax": true
+    "verbatimModuleSyntax": true,
+    "strict": true
   },
   "exclude": [
     "dist_*/**/*.d.ts"