From e9245111479887b6004a62ed10b7fb9092a5c9b2 Mon Sep 17 00:00:00 2001 From: Philipp Kunz Date: Mon, 27 May 2024 12:56:25 +0200 Subject: [PATCH] fix(s3 paths): pathing differences now correctly handled in a reducePath method. --- test/test.ts | 2 +- ts/00_commitinfo_data.ts | 2 +- ts/classes.bucket.ts | 151 ++++++++++++++++++++++++++++++++------- ts/classes.file.ts | 25 +++++++ ts/helpers.ts | 22 ++++++ ts/interfaces.ts | 6 ++ 6 files changed, 179 insertions(+), 29 deletions(-) create mode 100644 ts/helpers.ts create mode 100644 ts/interfaces.ts diff --git a/test/test.ts b/test/test.ts index f950167..d4fbbfa 100644 --- a/test/test.ts +++ b/test/test.ts @@ -79,7 +79,7 @@ tap.test('prepare for directory style tests', async () => { contents: 'dir3/dir4/file1.txt content', }); await myBucket.fastPut({ - path: 'file1.txt', + path: '/file1.txt', contents: 'file1 content', }); }); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index 98f99e8..6217307 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartbucket', - version: '3.0.6', + version: '3.0.7', description: 'A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.' } diff --git a/ts/classes.bucket.ts b/ts/classes.bucket.ts index c836507..e0dc6b7 100644 --- a/ts/classes.bucket.ts +++ b/ts/classes.bucket.ts @@ -1,6 +1,9 @@ import * as plugins from './plugins.js'; +import * as helpers from './helpers.js'; +import * as interfaces from './interfaces.js'; import { SmartBucket } from './classes.smartbucket.js'; import { Directory } from './classes.directory.js'; +import { File } from './classes.file.js'; export class Bucket { public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) { @@ -38,10 +41,19 @@ export class Bucket { /** * gets the base directory of the bucket */ - public async getBaseDirectory() { + public async getBaseDirectory(): Promise { return new Directory(this, null, ''); } + public async getDirectoryFromPath(pathDescriptorArg: interfaces.IPathDecriptor): Promise { + if (!pathDescriptorArg.path && !pathDescriptorArg.directory) { + return this.getBaseDirectory(); + } + let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg); + const baseDirectory = await this.getBaseDirectory(); + return await baseDirectory.getSubDirectoryByName(checkPath); + } + // =============== // Fast Operations // =============== @@ -53,28 +65,38 @@ export class Bucket { path: string; contents: string | Buffer; overwrite?: boolean; - }): Promise { + }): Promise { try { + const reducedPath = await helpers.reducePathDescriptorToPath({ + path: optionsArg.path, + }) // Check if the object already exists - const exists = await this.fastExists({ path: optionsArg.path }); + const exists = await this.fastExists({ path: reducedPath }); if (exists && !optionsArg.overwrite) { - console.error(`Object already exists at path '${optionsArg.path}' in bucket '${this.name}'.`); + console.error(`Object already exists at path '${reducedPath}' in bucket '${this.name}'.`); return; } else if (exists && optionsArg.overwrite) { - console.log(`Overwriting existing object at path '${optionsArg.path}' in bucket '${this.name}'.`); + console.log(`Overwriting existing object at path '${reducedPath}' in bucket '${this.name}'.`); } else { - console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`); + console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`); } // Proceed with putting the object const streamIntake = new plugins.smartstream.StreamIntake(); - const putPromise = this.smartbucketRef.minioClient.putObject(this.name, optionsArg.path, streamIntake); + const putPromise = this.smartbucketRef.minioClient.putObject(this.name, reducedPath, streamIntake); streamIntake.pushData(optionsArg.contents); streamIntake.signalEnd(); await putPromise; - console.log(`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`); + console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`); + const parsedPath = plugins.path.parse(reducedPath); + return new File({ + directoryRefArg: await this.getDirectoryFromPath({ + path: parsedPath.dir, + }), + fileName: parsedPath.base, + }); } catch (error) { console.error(`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`, error); throw error; @@ -183,19 +205,10 @@ export class Bucket { } - public async copyObject(optionsArg: { - /** - * the - */ - objectKey: string; - /** - * in case you want to copy to another bucket specify it here - */ + public async fastCopy(optionsArg: { + sourcePath: string; + destinationPath?: string; targetBucket?: Bucket; - targetBucketKey?: string; - /** - * metadata will be merged with existing metadata - */ nativeMetadata?: { [key: string]: string }; deleteExistingNativeMetadata?: boolean; }): Promise { @@ -205,7 +218,7 @@ export class Bucket { // Retrieve current object information to use in copy conditions const currentObjInfo = await this.smartbucketRef.minioClient.statObject( targetBucketName, - optionsArg.objectKey + optionsArg.sourcePath ); // Setting up copy conditions @@ -221,8 +234,8 @@ export class Bucket { // TODO: check on issue here: https://github.com/minio/minio-js/issues/1286 await this.smartbucketRef.minioClient.copyObject( this.name, - optionsArg.objectKey, - `/${targetBucketName}/${optionsArg.objectKey}`, + optionsArg.sourcePath, + `/${targetBucketName}/${optionsArg.destinationPath || optionsArg.sourcePath}`, copyConditions ); } catch (err) { @@ -231,6 +244,43 @@ export class Bucket { } } + /** + * Move object from one path to another within the same bucket or to another bucket + */ + public async fastMove(optionsArg: { + sourcePath: string; + destinationPath: string; + targetBucket?: Bucket; + overwrite?: boolean; + }): Promise { + try { + // Check if the destination object already exists + const destinationBucket = optionsArg.targetBucket || this; + const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath }); + + if (exists && !optionsArg.overwrite) { + console.error(`Object already exists at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`); + return; + } else if (exists && optionsArg.overwrite) { + console.log(`Overwriting existing object at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`); + } else { + console.log(`Moving object to path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`); + } + + // Proceed with copying the object to the new path + await this.fastCopy(optionsArg); + + // Remove the original object after successful copy + await this.fastRemove({ path: optionsArg.sourcePath }); + + console.log(`Object '${optionsArg.sourcePath}' has been successfully moved to '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`); + } catch (error) { + console.error(`Error moving object from '${optionsArg.sourcePath}' to '${optionsArg.destinationPath}':`, error); + throw error; + } + } + + /** * removeObject */ @@ -263,9 +313,56 @@ export class Bucket { } } - public async fastStat(optionsArg: { - path: string; - }) { - return this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path); + public async fastStat(pathDescriptor: interfaces.IPathDecriptor) { + let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor); + return this.smartbucketRef.minioClient.statObject(this.name, checkPath); + } + + public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise { + let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor); + + // lets check if the checkPath is a directory + const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true); + const done = plugins.smartpromise.defer(); + stream.on('data', (dataArg) => { + stream.destroy(); // Stop the stream early if we find at least one object + if (dataArg.prefix.startsWith(checkPath + '/')) { + done.resolve(true); + } + }); + + stream.on('end', () => { + done.resolve(false); + }); + + stream.on('error', (err) => { + done.reject(err); + }); + + return done.promise; + }; + + public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise { + let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor); + + // lets check if the checkPath is a directory + const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true); + const done = plugins.smartpromise.defer(); + stream.on('data', (dataArg) => { + stream.destroy(); // Stop the stream early if we find at least one object + if (dataArg.prefix === checkPath) { + done.resolve(true); + } + }); + + stream.on('end', () => { + done.resolve(false); + }); + + stream.on('error', (err) => { + done.reject(err); + }); + + return done.promise; } } diff --git a/ts/classes.file.ts b/ts/classes.file.ts index 6d6edba..1d61cbd 100644 --- a/ts/classes.file.ts +++ b/ts/classes.file.ts @@ -1,4 +1,6 @@ import * as plugins from './plugins.js'; +import * as helpers from './helpers.js'; +import * as interfaces from './interfaces.js'; import { Directory } from './classes.directory.js'; import { MetaData } from './classes.metadata.js'; @@ -144,6 +146,29 @@ export class File { } } + /** + * moves the file to another directory + */ + public async move(pathDescriptorArg: interfaces.IPathDecriptor) { + let moveToPath = ''; + const isDirectory = await this.parentDirectoryRef.bucketRef.isDirectory(pathDescriptorArg); + if (isDirectory) { + moveToPath = await helpers.reducePathDescriptorToPath({ + ...pathDescriptorArg, + path: plugins.path.join(pathDescriptorArg.path, this.name), + }); + } + // lets move the file + await this.parentDirectoryRef.bucketRef.fastMove({ + sourcePath: this.getBasePath(), + destinationPath: moveToPath, + }); + + // lets move the metadatafile + const metadata = await this.getMetaData(); + await metadata.metadataFile.move(pathDescriptorArg); + } + /** * allows updating the metadata of a file * @param updatedMetadata diff --git a/ts/helpers.ts b/ts/helpers.ts new file mode 100644 index 0000000..26d0693 --- /dev/null +++ b/ts/helpers.ts @@ -0,0 +1,22 @@ +import * as plugins from './plugins.js'; +import * as interfaces from './interfaces.js'; + +export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.IPathDecriptor): Promise => { + let returnPath = `` + if (pathDescriptorArg.directory) { + if (pathDescriptorArg.path && plugins.path.isAbsolute(pathDescriptorArg.path)) { + console.warn('Directory is being ignored when path is absolute.'); + returnPath = pathDescriptorArg.path; + } else if (pathDescriptorArg.path) { + returnPath = plugins.path.join(pathDescriptorArg.directory.getBasePath(), pathDescriptorArg.path); + } + } else if (pathDescriptorArg.path) { + returnPath = pathDescriptorArg.path; + } else { + throw new Error('You must specify either a path or a directory.'); + } + if (returnPath.startsWith('/')) { + returnPath = returnPath.substring(1); + } + return returnPath; +} \ No newline at end of file diff --git a/ts/interfaces.ts b/ts/interfaces.ts new file mode 100644 index 0000000..37345e4 --- /dev/null +++ b/ts/interfaces.ts @@ -0,0 +1,6 @@ +import type { Directory } from "./classes.directory.js"; + +export interface IPathDecriptor { + path?: string; + directory?: Directory; +} \ No newline at end of file