fix(core): update
This commit is contained in:
parent
8401fe1c0c
commit
535d9f8520
@ -19,6 +19,7 @@
|
|||||||
"@push.rocks/tapbundle": "^5.0.23"
|
"@push.rocks/tapbundle": "^5.0.23"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@aws-sdk/client-s3": "^3.598.0",
|
||||||
"@push.rocks/smartmime": "^2.0.2",
|
"@push.rocks/smartmime": "^2.0.2",
|
||||||
"@push.rocks/smartpath": "^5.0.18",
|
"@push.rocks/smartpath": "^5.0.18",
|
||||||
"@push.rocks/smartpromise": "^4.0.3",
|
"@push.rocks/smartpromise": "^4.0.3",
|
||||||
@ -26,8 +27,7 @@
|
|||||||
"@push.rocks/smartstream": "^3.0.44",
|
"@push.rocks/smartstream": "^3.0.44",
|
||||||
"@push.rocks/smartstring": "^4.0.15",
|
"@push.rocks/smartstring": "^4.0.15",
|
||||||
"@push.rocks/smartunique": "^3.0.9",
|
"@push.rocks/smartunique": "^3.0.9",
|
||||||
"@tsclass/tsclass": "^4.0.55",
|
"@tsclass/tsclass": "^4.0.59"
|
||||||
"minio": "^8.0.0"
|
|
||||||
},
|
},
|
||||||
"private": false,
|
"private": false,
|
||||||
"files": [
|
"files": [
|
||||||
|
1420
pnpm-lock.yaml
generated
1420
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartbucket',
|
name: '@push.rocks/smartbucket',
|
||||||
version: '3.0.15',
|
version: '3.0.16',
|
||||||
description: 'A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.'
|
description: 'A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.'
|
||||||
}
|
}
|
||||||
|
@ -1,3 +1,5 @@
|
|||||||
|
// classes.bucket.ts
|
||||||
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import * as helpers from './helpers.js';
|
import * as helpers from './helpers.js';
|
||||||
import * as interfaces from './interfaces.js';
|
import * as interfaces from './interfaces.js';
|
||||||
@ -7,16 +9,15 @@ import { File } from './classes.file.js';
|
|||||||
import { Trash } from './classes.trash.js';
|
import { Trash } from './classes.trash.js';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The bucket class exposes the basc functionality of a bucket.
|
* The bucket class exposes the basic functionality of a bucket.
|
||||||
* The functions of the bucket alone are enough to
|
* The functions of the bucket alone are enough to
|
||||||
* operate in s3 basic fashion on blobs of data.
|
* operate in S3 basic fashion on blobs of data.
|
||||||
*/
|
*/
|
||||||
export class Bucket {
|
export class Bucket {
|
||||||
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
|
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
|
||||||
const buckets = await smartbucketRef.minioClient.listBuckets();
|
const command = new plugins.s3.ListBucketsCommand({});
|
||||||
const foundBucket = buckets.find((bucket) => {
|
const buckets = await smartbucketRef.s3Client.send(command);
|
||||||
return bucket.name === bucketNameArg;
|
const foundBucket = buckets.Buckets.find((bucket) => bucket.Name === bucketNameArg);
|
||||||
});
|
|
||||||
|
|
||||||
if (foundBucket) {
|
if (foundBucket) {
|
||||||
console.log(`bucket with name ${bucketNameArg} exists.`);
|
console.log(`bucket with name ${bucketNameArg} exists.`);
|
||||||
@ -28,12 +29,14 @@ export class Bucket {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||||
await smartbucketRef.minioClient.makeBucket(bucketName, 'ams3').catch((e) => console.log(e));
|
const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName });
|
||||||
|
await smartbucketRef.s3Client.send(command).catch((e) => console.log(e));
|
||||||
return new Bucket(smartbucketRef, bucketName);
|
return new Bucket(smartbucketRef, bucketName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||||
await smartbucketRef.minioClient.removeBucket(bucketName).catch((e) => console.log(e));
|
const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName });
|
||||||
|
await smartbucketRef.s3Client.send(command).catch((e) => console.log(e));
|
||||||
}
|
}
|
||||||
|
|
||||||
public smartbucketRef: SmartBucket;
|
public smartbucketRef: SmartBucket;
|
||||||
@ -65,7 +68,7 @@ export class Bucket {
|
|||||||
if (!pathDescriptorArg.path && !pathDescriptorArg.directory) {
|
if (!pathDescriptorArg.path && !pathDescriptorArg.directory) {
|
||||||
return this.getBaseDirectory();
|
return this.getBaseDirectory();
|
||||||
}
|
}
|
||||||
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
|
const checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
|
||||||
const baseDirectory = await this.getBaseDirectory();
|
const baseDirectory = await this.getBaseDirectory();
|
||||||
return await baseDirectory.getSubDirectoryByName(checkPath);
|
return await baseDirectory.getSubDirectoryByName(checkPath);
|
||||||
}
|
}
|
||||||
@ -77,13 +80,14 @@ export class Bucket {
|
|||||||
/**
|
/**
|
||||||
* store file
|
* store file
|
||||||
*/
|
*/
|
||||||
public async fastPut(optionsArg: interfaces.IPathDecriptor & {
|
public async fastPut(
|
||||||
contents: string | Buffer;
|
optionsArg: interfaces.IPathDecriptor & {
|
||||||
overwrite?: boolean;
|
contents: string | Buffer;
|
||||||
}): Promise<File> {
|
overwrite?: boolean;
|
||||||
|
}
|
||||||
|
): Promise<File> {
|
||||||
try {
|
try {
|
||||||
const reducedPath = await helpers.reducePathDescriptorToPath(optionsArg);
|
const reducedPath = await helpers.reducePathDescriptorToPath(optionsArg);
|
||||||
// Check if the object already exists
|
|
||||||
const exists = await this.fastExists({ path: reducedPath });
|
const exists = await this.fastExists({ path: reducedPath });
|
||||||
|
|
||||||
if (exists && !optionsArg.overwrite) {
|
if (exists && !optionsArg.overwrite) {
|
||||||
@ -97,16 +101,12 @@ export class Bucket {
|
|||||||
console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`);
|
console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Proceed with putting the object
|
const command = new plugins.s3.PutObjectCommand({
|
||||||
const streamIntake = new plugins.smartstream.StreamIntake();
|
Bucket: this.name,
|
||||||
const putPromise = this.smartbucketRef.minioClient.putObject(
|
Key: reducedPath,
|
||||||
this.name,
|
Body: optionsArg.contents,
|
||||||
reducedPath,
|
});
|
||||||
streamIntake
|
await this.smartbucketRef.s3Client.send(command);
|
||||||
);
|
|
||||||
streamIntake.pushData(optionsArg.contents);
|
|
||||||
streamIntake.signalEnd();
|
|
||||||
await putPromise;
|
|
||||||
|
|
||||||
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
||||||
const parsedPath = plugins.path.parse(reducedPath);
|
const parsedPath = plugins.path.parse(reducedPath);
|
||||||
@ -161,27 +161,30 @@ export class Bucket {
|
|||||||
public async fastGetReplaySubject(optionsArg: {
|
public async fastGetReplaySubject(optionsArg: {
|
||||||
path: string;
|
path: string;
|
||||||
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
|
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
|
||||||
const fileStream = await this.smartbucketRef.minioClient
|
const command = new plugins.s3.GetObjectCommand({
|
||||||
.getObject(this.name, optionsArg.path)
|
Bucket: this.name,
|
||||||
.catch((e) => console.log(e));
|
Key: optionsArg.path,
|
||||||
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
|
||||||
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, void>({
|
|
||||||
writeFunction: async (chunk) => {
|
|
||||||
replaySubject.next(chunk);
|
|
||||||
return;
|
|
||||||
},
|
|
||||||
finalFunction: async (cb) => {
|
|
||||||
replaySubject.complete();
|
|
||||||
return;
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
const response = await this.smartbucketRef.s3Client.send(command);
|
||||||
|
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
||||||
|
|
||||||
if (!fileStream) {
|
// Convert the stream to a format that supports piping
|
||||||
return null;
|
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||||
|
if (typeof stream.pipe === 'function') {
|
||||||
|
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, void>({
|
||||||
|
writeFunction: async (chunk) => {
|
||||||
|
replaySubject.next(chunk);
|
||||||
|
return;
|
||||||
|
},
|
||||||
|
finalFunction: async (cb) => {
|
||||||
|
replaySubject.complete();
|
||||||
|
return;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
stream.pipe(duplexStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
const smartstream = new plugins.smartstream.StreamWrapper([fileStream, duplexStream]);
|
|
||||||
smartstream.run();
|
|
||||||
return replaySubject;
|
return replaySubject;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -198,18 +201,17 @@ export class Bucket {
|
|||||||
typeArg: 'nodestream'
|
typeArg: 'nodestream'
|
||||||
): Promise<plugins.stream.Readable>;
|
): Promise<plugins.stream.Readable>;
|
||||||
|
|
||||||
/**
|
|
||||||
* fastGetStream
|
|
||||||
* @param optionsArg
|
|
||||||
* @returns
|
|
||||||
*/
|
|
||||||
public async fastGetStream(
|
public async fastGetStream(
|
||||||
optionsArg: { path: string },
|
optionsArg: { path: string },
|
||||||
typeArg: 'webstream' | 'nodestream' = 'nodestream'
|
typeArg: 'webstream' | 'nodestream' = 'nodestream'
|
||||||
): Promise<ReadableStream | plugins.stream.Readable> {
|
): Promise<ReadableStream | plugins.stream.Readable> {
|
||||||
const fileStream = await this.smartbucketRef.minioClient
|
const command = new plugins.s3.GetObjectCommand({
|
||||||
.getObject(this.name, optionsArg.path)
|
Bucket: this.name,
|
||||||
.catch((e) => console.log(e));
|
Key: optionsArg.path,
|
||||||
|
});
|
||||||
|
const response = await this.smartbucketRef.s3Client.send(command);
|
||||||
|
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||||
|
|
||||||
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
||||||
writeFunction: async (chunk) => {
|
writeFunction: async (chunk) => {
|
||||||
return chunk;
|
return chunk;
|
||||||
@ -219,12 +221,10 @@ export class Bucket {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!fileStream) {
|
if (typeof stream.pipe === 'function') {
|
||||||
return null;
|
stream.pipe(duplexStream);
|
||||||
}
|
}
|
||||||
|
|
||||||
const smartstream = new plugins.smartstream.StreamWrapper([fileStream, duplexStream]);
|
|
||||||
smartstream.run();
|
|
||||||
if (typeArg === 'nodestream') {
|
if (typeArg === 'nodestream') {
|
||||||
return duplexStream;
|
return duplexStream;
|
||||||
}
|
}
|
||||||
@ -243,7 +243,6 @@ export class Bucket {
|
|||||||
overwrite?: boolean;
|
overwrite?: boolean;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Check if the object already exists
|
|
||||||
const exists = await this.fastExists({ path: optionsArg.path });
|
const exists = await this.fastExists({ path: optionsArg.path });
|
||||||
|
|
||||||
if (exists && !optionsArg.overwrite) {
|
if (exists && !optionsArg.overwrite) {
|
||||||
@ -259,18 +258,13 @@ export class Bucket {
|
|||||||
console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`);
|
console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const streamIntake = await plugins.smartstream.StreamIntake.fromStream<Uint8Array>(
|
const command = new plugins.s3.PutObjectCommand({
|
||||||
optionsArg.readableStream
|
Bucket: this.name,
|
||||||
);
|
Key: optionsArg.path,
|
||||||
|
Body: optionsArg.readableStream,
|
||||||
// Proceed with putting the object
|
Metadata: optionsArg.nativeMetadata,
|
||||||
await this.smartbucketRef.minioClient.putObject(
|
});
|
||||||
this.name,
|
await this.smartbucketRef.s3Client.send(command);
|
||||||
optionsArg.path,
|
|
||||||
streamIntake,
|
|
||||||
null,
|
|
||||||
null // TODO: Add support for custom metadata once proper support is in minio.
|
|
||||||
);
|
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
||||||
@ -295,28 +289,29 @@ export class Bucket {
|
|||||||
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
|
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
|
||||||
|
|
||||||
// Retrieve current object information to use in copy conditions
|
// Retrieve current object information to use in copy conditions
|
||||||
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
|
const currentObjInfo = await this.smartbucketRef.s3Client.send(
|
||||||
targetBucketName,
|
new plugins.s3.HeadObjectCommand({
|
||||||
optionsArg.sourcePath
|
Bucket: this.name,
|
||||||
|
Key: optionsArg.sourcePath,
|
||||||
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
// Setting up copy conditions
|
|
||||||
const copyConditions = new plugins.minio.CopyConditions();
|
|
||||||
|
|
||||||
// Prepare new metadata
|
// Prepare new metadata
|
||||||
const newNativeMetadata = {
|
const newNativeMetadata = {
|
||||||
...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.metaData),
|
...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.Metadata),
|
||||||
...optionsArg.nativeMetadata,
|
...optionsArg.nativeMetadata,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Define the copy operation as a Promise
|
// Define the copy operation
|
||||||
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
|
const copySource = `${this.name}/${optionsArg.sourcePath}`;
|
||||||
await this.smartbucketRef.minioClient.copyObject(
|
const command = new plugins.s3.CopyObjectCommand({
|
||||||
this.name,
|
Bucket: targetBucketName,
|
||||||
optionsArg.sourcePath,
|
CopySource: copySource,
|
||||||
`/${targetBucketName}/${optionsArg.destinationPath || optionsArg.sourcePath}`,
|
Key: optionsArg.destinationPath || optionsArg.sourcePath,
|
||||||
copyConditions
|
Metadata: newNativeMetadata,
|
||||||
);
|
MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
|
||||||
|
});
|
||||||
|
await this.smartbucketRef.s3Client.send(command);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error updating metadata:', err);
|
console.error('Error updating metadata:', err);
|
||||||
throw err; // rethrow to allow caller to handle
|
throw err; // rethrow to allow caller to handle
|
||||||
@ -333,7 +328,6 @@ export class Bucket {
|
|||||||
overwrite?: boolean;
|
overwrite?: boolean;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Check if the destination object already exists
|
|
||||||
const destinationBucket = optionsArg.targetBucket || this;
|
const destinationBucket = optionsArg.targetBucket || this;
|
||||||
const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath });
|
const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath });
|
||||||
|
|
||||||
@ -352,10 +346,7 @@ export class Bucket {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Proceed with copying the object to the new path
|
|
||||||
await this.fastCopy(optionsArg);
|
await this.fastCopy(optionsArg);
|
||||||
|
|
||||||
// Remove the original object after successful copy
|
|
||||||
await this.fastRemove({ path: optionsArg.sourcePath });
|
await this.fastRemove({ path: optionsArg.sourcePath });
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
@ -374,21 +365,29 @@ export class Bucket {
|
|||||||
* removeObject
|
* removeObject
|
||||||
*/
|
*/
|
||||||
public async fastRemove(optionsArg: { path: string }) {
|
public async fastRemove(optionsArg: { path: string }) {
|
||||||
await this.smartbucketRef.minioClient.removeObject(this.name, optionsArg.path);
|
const command = new plugins.s3.DeleteObjectCommand({
|
||||||
|
Bucket: this.name,
|
||||||
|
Key: optionsArg.path,
|
||||||
|
});
|
||||||
|
await this.smartbucketRef.s3Client.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* check wether file exists
|
* check whether file exists
|
||||||
* @param optionsArg
|
* @param optionsArg
|
||||||
* @returns
|
* @returns
|
||||||
*/
|
*/
|
||||||
public async fastExists(optionsArg: { path: string }): Promise<boolean> {
|
public async fastExists(optionsArg: { path: string }): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
await this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path);
|
const command = new plugins.s3.HeadObjectCommand({
|
||||||
|
Bucket: this.name,
|
||||||
|
Key: optionsArg.path,
|
||||||
|
});
|
||||||
|
await this.smartbucketRef.s3Client.send(command);
|
||||||
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
||||||
return true;
|
return true;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error.code === 'NotFound') {
|
if (error.name === 'NotFound') {
|
||||||
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
|
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
|
||||||
return false;
|
return false;
|
||||||
} else {
|
} else {
|
||||||
@ -402,59 +401,39 @@ export class Bucket {
|
|||||||
* deletes this bucket
|
* deletes this bucket
|
||||||
*/
|
*/
|
||||||
public async delete() {
|
public async delete() {
|
||||||
await this.smartbucketRef.minioClient.removeBucket(this.name);
|
await this.smartbucketRef.s3Client.send(
|
||||||
|
new plugins.s3.DeleteBucketCommand({ Bucket: this.name })
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async fastStat(pathDescriptor: interfaces.IPathDecriptor) {
|
public async fastStat(pathDescriptor: interfaces.IPathDecriptor) {
|
||||||
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
const checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
||||||
return this.smartbucketRef.minioClient.statObject(this.name, checkPath);
|
const command = new plugins.s3.HeadObjectCommand({
|
||||||
|
Bucket: this.name,
|
||||||
|
Key: checkPath,
|
||||||
|
});
|
||||||
|
return this.smartbucketRef.s3Client.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
||||||
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
const checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
||||||
|
const command = new plugins.s3.ListObjectsV2Command({
|
||||||
// lets check if the checkPath is a directory
|
Bucket: this.name,
|
||||||
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
|
Prefix: checkPath,
|
||||||
const done = plugins.smartpromise.defer<boolean>();
|
Delimiter: '/',
|
||||||
stream.on('data', (dataArg) => {
|
|
||||||
stream.destroy(); // Stop the stream early if we find at least one object
|
|
||||||
if (dataArg.prefix.startsWith(checkPath + '/')) {
|
|
||||||
done.resolve(true);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
const response = await this.smartbucketRef.s3Client.send(command);
|
||||||
stream.on('end', () => {
|
return response.CommonPrefixes.length > 0;
|
||||||
done.resolve(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
stream.on('error', (err) => {
|
|
||||||
done.reject(err);
|
|
||||||
});
|
|
||||||
|
|
||||||
return done.promise;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
||||||
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
const checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
|
||||||
|
const command = new plugins.s3.ListObjectsV2Command({
|
||||||
// lets check if the checkPath is a directory
|
Bucket: this.name,
|
||||||
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
|
Prefix: checkPath,
|
||||||
const done = plugins.smartpromise.defer<boolean>();
|
Delimiter: '/',
|
||||||
stream.on('data', (dataArg) => {
|
|
||||||
stream.destroy(); // Stop the stream early if we find at least one object
|
|
||||||
if (dataArg.prefix === checkPath) {
|
|
||||||
done.resolve(true);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
const response = await this.smartbucketRef.s3Client.send(command);
|
||||||
stream.on('end', () => {
|
return response.Contents.length > 0;
|
||||||
done.resolve(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
stream.on('error', (err) => {
|
|
||||||
done.reject(err);
|
|
||||||
});
|
|
||||||
|
|
||||||
return done.promise;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,8 @@
|
|||||||
|
// classes.directory.ts
|
||||||
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import { Bucket } from './classes.bucket.js';
|
import { Bucket } from './classes.bucket.js';
|
||||||
import { File } from './classes.file.js';
|
import { File } from './classes.file.js';
|
||||||
|
|
||||||
import * as helpers from './helpers.js';
|
import * as helpers from './helpers.js';
|
||||||
|
|
||||||
export class Directory {
|
export class Directory {
|
||||||
@ -13,9 +14,9 @@ export class Directory {
|
|||||||
public files: string[];
|
public files: string[];
|
||||||
public folders: string[];
|
public folders: string[];
|
||||||
|
|
||||||
constructor(bucketRefArg: Bucket, parentDiretory: Directory, name: string) {
|
constructor(bucketRefArg: Bucket, parentDirectory: Directory, name: string) {
|
||||||
this.bucketRef = bucketRefArg;
|
this.bucketRef = bucketRefArg;
|
||||||
this.parentDirectoryRef = parentDiretory;
|
this.parentDirectoryRef = parentDirectory;
|
||||||
this.name = name;
|
this.name = name;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -73,15 +74,12 @@ export class Directory {
|
|||||||
directory: this,
|
directory: this,
|
||||||
path: optionsArg.name,
|
path: optionsArg.name,
|
||||||
};
|
};
|
||||||
// check wether the file exists
|
|
||||||
const exists = await this.bucketRef.fastExists({
|
const exists = await this.bucketRef.fastExists({
|
||||||
path: await helpers.reducePathDescriptorToPath(pathDescriptor),
|
path: await helpers.reducePathDescriptorToPath(pathDescriptor),
|
||||||
});
|
});
|
||||||
if (!exists && optionsArg.getFromTrash) {
|
if (!exists && optionsArg.getFromTrash) {
|
||||||
const trash = await this.bucketRef.getTrash();
|
const trash = await this.bucketRef.getTrash();
|
||||||
const trashedFile = await trash.getTrashedFileByOriginalName(
|
const trashedFile = await trash.getTrashedFileByOriginalName(pathDescriptor);
|
||||||
pathDescriptor
|
|
||||||
)
|
|
||||||
return trashedFile;
|
return trashedFile;
|
||||||
}
|
}
|
||||||
if (!exists && !optionsArg.createWithContents) {
|
if (!exists && !optionsArg.createWithContents) {
|
||||||
@ -104,26 +102,17 @@ export class Directory {
|
|||||||
* lists all files
|
* lists all files
|
||||||
*/
|
*/
|
||||||
public async listFiles(): Promise<File[]> {
|
public async listFiles(): Promise<File[]> {
|
||||||
const done = plugins.smartpromise.defer();
|
const command = new plugins.s3.ListObjectsV2Command({
|
||||||
const fileNameStream = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
|
Bucket: this.bucketRef.name,
|
||||||
this.bucketRef.name,
|
Prefix: this.getBasePath(),
|
||||||
this.getBasePath(),
|
Delimiter: '/',
|
||||||
false
|
});
|
||||||
);
|
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||||
const fileArray: File[] = [];
|
const fileArray: File[] = [];
|
||||||
const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
|
|
||||||
objectMode: true,
|
response.Contents.forEach((item) => {
|
||||||
writeFunction: async (bucketItem) => {
|
if (item.Key && !item.Key.endsWith('/')) {
|
||||||
if (bucketItem.prefix) {
|
const subtractedPath = item.Key.replace(this.getBasePath(), '');
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!bucketItem.name) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let subtractedPath = bucketItem.name.replace(this.getBasePath(), '');
|
|
||||||
if (subtractedPath.startsWith('/')) {
|
|
||||||
subtractedPath = subtractedPath.substr(1);
|
|
||||||
}
|
|
||||||
if (!subtractedPath.includes('/')) {
|
if (!subtractedPath.includes('/')) {
|
||||||
fileArray.push(
|
fileArray.push(
|
||||||
new File({
|
new File({
|
||||||
@ -132,13 +121,9 @@ export class Directory {
|
|||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
finalFunction: async (tools) => {
|
|
||||||
done.resolve();
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
fileNameStream.pipe(duplexStream);
|
|
||||||
await done.promise;
|
|
||||||
return fileArray;
|
return fileArray;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -146,54 +131,52 @@ export class Directory {
|
|||||||
* lists all folders
|
* lists all folders
|
||||||
*/
|
*/
|
||||||
public async listDirectories(): Promise<Directory[]> {
|
public async listDirectories(): Promise<Directory[]> {
|
||||||
const done = plugins.smartpromise.defer();
|
try {
|
||||||
const basePath = this.getBasePath();
|
const command = new plugins.s3.ListObjectsV2Command({
|
||||||
const completeDirStream = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
|
Bucket: this.bucketRef.name,
|
||||||
this.bucketRef.name,
|
Prefix: this.getBasePath(),
|
||||||
this.getBasePath(),
|
Delimiter: '/',
|
||||||
false
|
});
|
||||||
);
|
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||||
const directoryArray: Directory[] = [];
|
const directoryArray: Directory[] = [];
|
||||||
const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
|
|
||||||
objectMode: true,
|
if (response.CommonPrefixes) {
|
||||||
writeFunction: async (bucketItem) => {
|
response.CommonPrefixes.forEach((item) => {
|
||||||
if (bucketItem.name) {
|
if (item.Prefix) {
|
||||||
return;
|
const subtractedPath = item.Prefix.replace(this.getBasePath(), '');
|
||||||
}
|
if (subtractedPath.endsWith('/')) {
|
||||||
let subtractedPath = bucketItem.prefix.replace(this.getBasePath(), '');
|
const dirName = subtractedPath.slice(0, -1);
|
||||||
if (subtractedPath.startsWith('/')) {
|
// Ensure the directory name is not empty (which would indicate the base directory itself)
|
||||||
subtractedPath = subtractedPath.substr(1);
|
if (dirName) {
|
||||||
}
|
directoryArray.push(new Directory(this.bucketRef, this, dirName));
|
||||||
if (subtractedPath.includes('/')) {
|
}
|
||||||
const dirName = subtractedPath.split('/')[0];
|
}
|
||||||
if (directoryArray.find((directory) => directory.name === dirName)) {
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
directoryArray.push(new Directory(this.bucketRef, this, dirName));
|
});
|
||||||
}
|
}
|
||||||
},
|
|
||||||
finalFunction: async (tools) => {
|
return directoryArray;
|
||||||
done.resolve();
|
} catch (error) {
|
||||||
},
|
console.error('Error listing directories:', error);
|
||||||
});
|
throw error;
|
||||||
completeDirStream.pipe(duplexStream);
|
}
|
||||||
await done.promise;
|
|
||||||
return directoryArray;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gets an array that has all objects with a certain prefix;
|
* gets an array that has all objects with a certain prefix
|
||||||
*/
|
*/
|
||||||
public async getTreeArray() {
|
public async getTreeArray() {
|
||||||
const treeArray = await this.bucketRef.smartbucketRef.minioClient.listObjectsV2(
|
const command = new plugins.s3.ListObjectsV2Command({
|
||||||
this.bucketRef.name,
|
Bucket: this.bucketRef.name,
|
||||||
this.getBasePath(),
|
Prefix: this.getBasePath(),
|
||||||
true
|
Delimiter: '/',
|
||||||
);
|
});
|
||||||
|
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||||
|
return response.Contents;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* gets a sub directory
|
* gets a sub directory by name
|
||||||
*/
|
*/
|
||||||
public async getSubDirectoryByName(dirNameArg: string): Promise<Directory> {
|
public async getSubDirectoryByName(dirNameArg: string): Promise<Directory> {
|
||||||
const dirNameArray = dirNameArg.split('/');
|
const dirNameArray = dirNameArg.split('/');
|
||||||
@ -204,11 +187,13 @@ export class Directory {
|
|||||||
return directory.name === dirNameToSearch;
|
return directory.name === dirNameToSearch;
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
let wantedDirectory: Directory;
|
let wantedDirectory: Directory;
|
||||||
for (const dirNameToSearch of dirNameArray) {
|
for (const dirNameToSearch of dirNameArray) {
|
||||||
const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
|
const directoryToSearchIn = wantedDirectory ? wantedDirectory : this;
|
||||||
wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch);
|
wantedDirectory = await getDirectory(directoryToSearchIn, dirNameToSearch);
|
||||||
}
|
}
|
||||||
|
|
||||||
return wantedDirectory;
|
return wantedDirectory;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -217,19 +202,20 @@ export class Directory {
|
|||||||
*/
|
*/
|
||||||
public async move() {
|
public async move() {
|
||||||
// TODO
|
// TODO
|
||||||
throw new Error('moving a directory is not yet implemented');
|
throw new Error('Moving a directory is not yet implemented');
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* creates a file within this directory
|
* creates an empty file within this directory
|
||||||
* @param relativePathArg
|
* @param relativePathArg
|
||||||
*/
|
*/
|
||||||
public async createEmptyFile(relativePathArg: string) {
|
public async createEmptyFile(relativePathArg: string) {
|
||||||
const emtpyFile = await File.create({
|
const emptyFile = await File.create({
|
||||||
directory: this,
|
directory: this,
|
||||||
name: relativePathArg,
|
name: relativePathArg,
|
||||||
contents: '',
|
contents: '',
|
||||||
});
|
});
|
||||||
|
return emptyFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
// file operations
|
// file operations
|
||||||
@ -313,7 +299,7 @@ export class Directory {
|
|||||||
const deleteDirectory = async (directoryArg: Directory) => {
|
const deleteDirectory = async (directoryArg: Directory) => {
|
||||||
const childDirectories = await directoryArg.listDirectories();
|
const childDirectories = await directoryArg.listDirectories();
|
||||||
if (childDirectories.length === 0) {
|
if (childDirectories.length === 0) {
|
||||||
console.log('directory empty! Path complete!');
|
console.log('Directory empty! Path complete!');
|
||||||
} else {
|
} else {
|
||||||
for (const childDir of childDirectories) {
|
for (const childDir of childDirectories) {
|
||||||
await deleteDirectory(childDir);
|
await deleteDirectory(childDir);
|
||||||
|
@ -44,7 +44,7 @@ export class MetaData {
|
|||||||
const stat = await this.fileRef.parentDirectoryRef.bucketRef.fastStat({
|
const stat = await this.fileRef.parentDirectoryRef.bucketRef.fastStat({
|
||||||
path: this.fileRef.getBasePath(),
|
path: this.fileRef.getBasePath(),
|
||||||
});
|
});
|
||||||
return stat.size;
|
return stat.ContentLength;
|
||||||
}
|
}
|
||||||
|
|
||||||
private prefixCustomMetaData = 'custom_';
|
private prefixCustomMetaData = 'custom_';
|
||||||
|
@ -1,22 +1,33 @@
|
|||||||
|
// classes.smartbucket.ts
|
||||||
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import { Bucket } from './classes.bucket.js';
|
import { Bucket } from './classes.bucket.js';
|
||||||
|
|
||||||
export class SmartBucket {
|
export class SmartBucket {
|
||||||
public config: plugins.tsclass.storage.IS3Descriptor;
|
public config: plugins.tsclass.storage.IS3Descriptor;
|
||||||
|
|
||||||
public minioClient: plugins.minio.Client;
|
public s3Client: plugins.s3.S3Client;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* the constructor of SmartBucket
|
||||||
|
*/
|
||||||
/**
|
/**
|
||||||
* the constructor of SmartBucket
|
* the constructor of SmartBucket
|
||||||
*/
|
*/
|
||||||
constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
|
constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
|
||||||
this.config = configArg;
|
this.config = configArg;
|
||||||
this.minioClient = new plugins.minio.Client({
|
const endpoint = this.config.endpoint.startsWith('http://') || this.config.endpoint.startsWith('https://')
|
||||||
endPoint: this.config.endpoint,
|
? this.config.endpoint
|
||||||
port: configArg.port || 443,
|
: `https://${this.config.endpoint}`;
|
||||||
useSSL: configArg.useSsl !== undefined ? configArg.useSsl : true,
|
|
||||||
accessKey: this.config.accessKey,
|
this.s3Client = new plugins.s3.S3Client({
|
||||||
secretKey: this.config.accessSecret,
|
endpoint,
|
||||||
|
region: this.config.region || 'us-east-1',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: this.config.accessKey,
|
||||||
|
secretAccessKey: this.config.accessSecret,
|
||||||
|
},
|
||||||
|
forcePathStyle: true, // Necessary for S3-compatible storage like MinIO or Wasabi
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -32,4 +43,4 @@ export class SmartBucket {
|
|||||||
public async getBucketByName(bucketName: string) {
|
public async getBucketByName(bucketName: string) {
|
||||||
return Bucket.getBucketByName(this, bucketName);
|
return Bucket.getBucketByName(this, bucketName);
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -1,3 +1,5 @@
|
|||||||
|
// plugins.ts
|
||||||
|
|
||||||
// node native
|
// node native
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import * as stream from 'stream';
|
import * as stream from 'stream';
|
||||||
@ -23,6 +25,8 @@ export {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// third party scope
|
// third party scope
|
||||||
import * as minio from 'minio';
|
import * as s3 from '@aws-sdk/client-s3';
|
||||||
|
|
||||||
export { minio };
|
export {
|
||||||
|
s3,
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user