fix(s3 paths): pathing differences now correctly handled in a reducePath method.

This commit is contained in:
Philipp Kunz 2024-05-27 12:56:25 +02:00
parent 645ebbdd4d
commit e924511147
6 changed files with 179 additions and 29 deletions

View File

@ -79,7 +79,7 @@ tap.test('prepare for directory style tests', async () => {
contents: 'dir3/dir4/file1.txt content', contents: 'dir3/dir4/file1.txt content',
}); });
await myBucket.fastPut({ await myBucket.fastPut({
path: 'file1.txt', path: '/file1.txt',
contents: 'file1 content', contents: 'file1 content',
}); });
}); });

View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartbucket', name: '@push.rocks/smartbucket',
version: '3.0.6', version: '3.0.7',
description: 'A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.' description: 'A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.'
} }

View File

@ -1,6 +1,9 @@
import * as plugins from './plugins.js'; import * as plugins from './plugins.js';
import * as helpers from './helpers.js';
import * as interfaces from './interfaces.js';
import { SmartBucket } from './classes.smartbucket.js'; import { SmartBucket } from './classes.smartbucket.js';
import { Directory } from './classes.directory.js'; import { Directory } from './classes.directory.js';
import { File } from './classes.file.js';
export class Bucket { export class Bucket {
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) { public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
@ -38,10 +41,19 @@ export class Bucket {
/** /**
* gets the base directory of the bucket * gets the base directory of the bucket
*/ */
public async getBaseDirectory() { public async getBaseDirectory(): Promise<Directory> {
return new Directory(this, null, ''); return new Directory(this, null, '');
} }
public async getDirectoryFromPath(pathDescriptorArg: interfaces.IPathDecriptor): Promise<Directory> {
if (!pathDescriptorArg.path && !pathDescriptorArg.directory) {
return this.getBaseDirectory();
}
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
const baseDirectory = await this.getBaseDirectory();
return await baseDirectory.getSubDirectoryByName(checkPath);
}
// =============== // ===============
// Fast Operations // Fast Operations
// =============== // ===============
@ -53,28 +65,38 @@ export class Bucket {
path: string; path: string;
contents: string | Buffer; contents: string | Buffer;
overwrite?: boolean; overwrite?: boolean;
}): Promise<void> { }): Promise<File> {
try { try {
const reducedPath = await helpers.reducePathDescriptorToPath({
path: optionsArg.path,
})
// Check if the object already exists // Check if the object already exists
const exists = await this.fastExists({ path: optionsArg.path }); const exists = await this.fastExists({ path: reducedPath });
if (exists && !optionsArg.overwrite) { if (exists && !optionsArg.overwrite) {
console.error(`Object already exists at path '${optionsArg.path}' in bucket '${this.name}'.`); console.error(`Object already exists at path '${reducedPath}' in bucket '${this.name}'.`);
return; return;
} else if (exists && optionsArg.overwrite) { } else if (exists && optionsArg.overwrite) {
console.log(`Overwriting existing object at path '${optionsArg.path}' in bucket '${this.name}'.`); console.log(`Overwriting existing object at path '${reducedPath}' in bucket '${this.name}'.`);
} else { } else {
console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`); console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`);
} }
// Proceed with putting the object // Proceed with putting the object
const streamIntake = new plugins.smartstream.StreamIntake(); const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = this.smartbucketRef.minioClient.putObject(this.name, optionsArg.path, streamIntake); const putPromise = this.smartbucketRef.minioClient.putObject(this.name, reducedPath, streamIntake);
streamIntake.pushData(optionsArg.contents); streamIntake.pushData(optionsArg.contents);
streamIntake.signalEnd(); streamIntake.signalEnd();
await putPromise; await putPromise;
console.log(`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`); console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
const parsedPath = plugins.path.parse(reducedPath);
return new File({
directoryRefArg: await this.getDirectoryFromPath({
path: parsedPath.dir,
}),
fileName: parsedPath.base,
});
} catch (error) { } catch (error) {
console.error(`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`, error); console.error(`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`, error);
throw error; throw error;
@ -183,19 +205,10 @@ export class Bucket {
} }
public async copyObject(optionsArg: { public async fastCopy(optionsArg: {
/** sourcePath: string;
* the destinationPath?: string;
*/
objectKey: string;
/**
* in case you want to copy to another bucket specify it here
*/
targetBucket?: Bucket; targetBucket?: Bucket;
targetBucketKey?: string;
/**
* metadata will be merged with existing metadata
*/
nativeMetadata?: { [key: string]: string }; nativeMetadata?: { [key: string]: string };
deleteExistingNativeMetadata?: boolean; deleteExistingNativeMetadata?: boolean;
}): Promise<void> { }): Promise<void> {
@ -205,7 +218,7 @@ export class Bucket {
// Retrieve current object information to use in copy conditions // Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.minioClient.statObject( const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
targetBucketName, targetBucketName,
optionsArg.objectKey optionsArg.sourcePath
); );
// Setting up copy conditions // Setting up copy conditions
@ -221,8 +234,8 @@ export class Bucket {
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286 // TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
await this.smartbucketRef.minioClient.copyObject( await this.smartbucketRef.minioClient.copyObject(
this.name, this.name,
optionsArg.objectKey, optionsArg.sourcePath,
`/${targetBucketName}/${optionsArg.objectKey}`, `/${targetBucketName}/${optionsArg.destinationPath || optionsArg.sourcePath}`,
copyConditions copyConditions
); );
} catch (err) { } catch (err) {
@ -231,6 +244,43 @@ export class Bucket {
} }
} }
/**
* Move object from one path to another within the same bucket or to another bucket
*/
public async fastMove(optionsArg: {
sourcePath: string;
destinationPath: string;
targetBucket?: Bucket;
overwrite?: boolean;
}): Promise<void> {
try {
// Check if the destination object already exists
const destinationBucket = optionsArg.targetBucket || this;
const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath });
if (exists && !optionsArg.overwrite) {
console.error(`Object already exists at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
return;
} else if (exists && optionsArg.overwrite) {
console.log(`Overwriting existing object at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
} else {
console.log(`Moving object to path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
}
// Proceed with copying the object to the new path
await this.fastCopy(optionsArg);
// Remove the original object after successful copy
await this.fastRemove({ path: optionsArg.sourcePath });
console.log(`Object '${optionsArg.sourcePath}' has been successfully moved to '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
} catch (error) {
console.error(`Error moving object from '${optionsArg.sourcePath}' to '${optionsArg.destinationPath}':`, error);
throw error;
}
}
/** /**
* removeObject * removeObject
*/ */
@ -263,9 +313,56 @@ export class Bucket {
} }
} }
public async fastStat(optionsArg: { public async fastStat(pathDescriptor: interfaces.IPathDecriptor) {
path: string; let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
}) { return this.smartbucketRef.minioClient.statObject(this.name, checkPath);
return this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path); }
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
// lets check if the checkPath is a directory
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
const done = plugins.smartpromise.defer<boolean>();
stream.on('data', (dataArg) => {
stream.destroy(); // Stop the stream early if we find at least one object
if (dataArg.prefix.startsWith(checkPath + '/')) {
done.resolve(true);
}
});
stream.on('end', () => {
done.resolve(false);
});
stream.on('error', (err) => {
done.reject(err);
});
return done.promise;
};
public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
// lets check if the checkPath is a directory
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
const done = plugins.smartpromise.defer<boolean>();
stream.on('data', (dataArg) => {
stream.destroy(); // Stop the stream early if we find at least one object
if (dataArg.prefix === checkPath) {
done.resolve(true);
}
});
stream.on('end', () => {
done.resolve(false);
});
stream.on('error', (err) => {
done.reject(err);
});
return done.promise;
} }
} }

View File

@ -1,4 +1,6 @@
import * as plugins from './plugins.js'; import * as plugins from './plugins.js';
import * as helpers from './helpers.js';
import * as interfaces from './interfaces.js';
import { Directory } from './classes.directory.js'; import { Directory } from './classes.directory.js';
import { MetaData } from './classes.metadata.js'; import { MetaData } from './classes.metadata.js';
@ -144,6 +146,29 @@ export class File {
} }
} }
/**
* moves the file to another directory
*/
public async move(pathDescriptorArg: interfaces.IPathDecriptor) {
let moveToPath = '';
const isDirectory = await this.parentDirectoryRef.bucketRef.isDirectory(pathDescriptorArg);
if (isDirectory) {
moveToPath = await helpers.reducePathDescriptorToPath({
...pathDescriptorArg,
path: plugins.path.join(pathDescriptorArg.path, this.name),
});
}
// lets move the file
await this.parentDirectoryRef.bucketRef.fastMove({
sourcePath: this.getBasePath(),
destinationPath: moveToPath,
});
// lets move the metadatafile
const metadata = await this.getMetaData();
await metadata.metadataFile.move(pathDescriptorArg);
}
/** /**
* allows updating the metadata of a file * allows updating the metadata of a file
* @param updatedMetadata * @param updatedMetadata

22
ts/helpers.ts Normal file
View File

@ -0,0 +1,22 @@
import * as plugins from './plugins.js';
import * as interfaces from './interfaces.js';
export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.IPathDecriptor): Promise<string> => {
let returnPath = ``
if (pathDescriptorArg.directory) {
if (pathDescriptorArg.path && plugins.path.isAbsolute(pathDescriptorArg.path)) {
console.warn('Directory is being ignored when path is absolute.');
returnPath = pathDescriptorArg.path;
} else if (pathDescriptorArg.path) {
returnPath = plugins.path.join(pathDescriptorArg.directory.getBasePath(), pathDescriptorArg.path);
}
} else if (pathDescriptorArg.path) {
returnPath = pathDescriptorArg.path;
} else {
throw new Error('You must specify either a path or a directory.');
}
if (returnPath.startsWith('/')) {
returnPath = returnPath.substring(1);
}
return returnPath;
}

6
ts/interfaces.ts Normal file
View File

@ -0,0 +1,6 @@
import type { Directory } from "./classes.directory.js";
export interface IPathDecriptor {
path?: string;
directory?: Directory;
}