Compare commits

...

10 Commits

Author SHA1 Message Date
2a17ee542e 3.0.8 2024-05-27 14:34:13 +02:00
95e9d2f0ff fix(core): update 2024-05-27 14:34:12 +02:00
1a71c76da3 3.0.7 2024-05-27 12:56:26 +02:00
e924511147 fix(s3 paths): pathing differences now correctly handled in a reducePath method. 2024-05-27 12:56:25 +02:00
645ebbdd4d 3.0.6 2024-05-21 18:47:00 +02:00
168148b2c9 fix(core): update 2024-05-21 18:46:59 +02:00
1293fc4ca6 3.0.5 2024-05-21 18:42:55 +02:00
b040120813 fix(core): update 2024-05-21 18:42:55 +02:00
5c2d92c041 3.0.4 2024-05-21 01:22:22 +02:00
eaf2e7e6bb fix(core): update 2024-05-21 01:22:21 +02:00
18 changed files with 827 additions and 379 deletions

View File

@ -8,7 +8,7 @@
"githost": "code.foss.global",
"gitscope": "push.rocks",
"gitrepo": "smartbucket",
"description": "A TypeScript library that offers simple, cloud-independent object storage with features like bucket creation, file management, and directory management.",
"description": "A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.",
"npmPackagename": "@push.rocks/smartbucket",
"license": "MIT",
"keywords": [
@ -26,7 +26,12 @@
"unified storage",
"buffer handling",
"access key",
"secret key"
"secret key",
"metadata",
"file locking",
"file streaming",
"directory listing",
"cloud agnostic"
]
}
},

4
package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@push.rocks/smartbucket",
"version": "3.0.3",
"version": "3.0.8",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@push.rocks/smartbucket",
"version": "3.0.3",
"version": "3.0.8",
"license": "UNLICENSED",
"dependencies": {
"@push.rocks/smartpath": "^5.0.18",

View File

@ -1,7 +1,7 @@
{
"name": "@push.rocks/smartbucket",
"version": "3.0.3",
"description": "A TypeScript library that offers simple, cloud-independent object storage with features like bucket creation, file management, and directory management.",
"version": "3.0.8",
"description": "A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.",
"main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts",
"type": "module",
@ -19,6 +19,7 @@
"@push.rocks/tapbundle": "^5.0.23"
},
"dependencies": {
"@push.rocks/smartmime": "^2.0.0",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrx": "^3.0.7",
@ -57,6 +58,11 @@
"unified storage",
"buffer handling",
"access key",
"secret key"
"secret key",
"metadata",
"file locking",
"file streaming",
"directory listing",
"cloud agnostic"
]
}

66
pnpm-lock.yaml generated
View File

@ -8,6 +8,9 @@ importers:
.:
dependencies:
'@push.rocks/smartmime':
specifier: ^2.0.0
version: 2.0.0
'@push.rocks/smartpath':
specifier: ^5.0.18
version: 5.0.18
@ -409,6 +412,9 @@ packages:
'@push.rocks/smartmime@1.0.6':
resolution: {integrity: sha512-PHd+I4UcsnOATNg8wjDsSAmmJ4CwQFrQCNzd0HSJMs4ZpiK3Ya91almd6GLpDPU370U4HFh4FaPF4eEAI6vkJQ==}
'@push.rocks/smartmime@2.0.0':
resolution: {integrity: sha512-yNEYrQzWjxwinCT8djw9eFumpCIvIQQS9KXWLH0LT9COlFoaP/ruk7pogrGYKCo20tFITJyO6NmMCa24402rvA==}
'@push.rocks/smartnetwork@3.0.2':
resolution: {integrity: sha512-s6CNGzQ1n/d/6cOKXbxeW6/tO//dr1woLqI01g7XhqTriw0nsm2G2kWaZh2J0VOguGNWBgQVCIpR0LjdRNWb3g==}
@ -609,6 +615,9 @@ packages:
'@tempfix/watcher@2.3.0':
resolution: {integrity: sha512-a2qVQffcrnetehvwsN+LdipxQ6jejwZLgAvS9/91+C0gP4CKyikY01c0tSs0I4tSL7qHdCw1Fx0quLw+A9uyLA==}
'@tokenizer/token@0.3.0':
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
'@tsclass/tsclass@3.0.48':
resolution: {integrity: sha512-hC65UvDlp9qvsl6OcIZXz0JNiWZ0gyzsTzbXpg215sGxopgbkOLCr6E0s4qCTnweYm95gt2AdY95uP7M7kExaQ==}
@ -1445,6 +1454,10 @@ packages:
resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==}
engines: {node: '>=18'}
file-type@19.0.0:
resolution: {integrity: sha512-s7cxa7/leUWLiXO78DVVfBVse+milos9FitauDLG1pI7lNaJ2+5lzPnr2N24ym+84HVwJL6hVuGfgVE+ALvU8Q==}
engines: {node: '>=18'}
fill-range@7.0.1:
resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==}
engines: {node: '>=8'}
@ -2142,6 +2155,11 @@ packages:
engines: {node: '>=4'}
hasBin: true
mime@4.0.3:
resolution: {integrity: sha512-KgUb15Oorc0NEKPbvfa0wRU+PItIEZmiv+pyAO2i0oTIVTJhlzMclU7w4RXWQrSOVH5ax/p/CkIO7KI4OyFJTQ==}
engines: {node: '>=16'}
hasBin: true
mimic-fn@2.1.0:
resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==}
engines: {node: '>=6'}
@ -2358,6 +2376,10 @@ packages:
resolution: {integrity: sha512-AHXsYi9EcYlSm3uUANz7h5WSktHiyTnUeHqdWmyRdjdMhgq9LgZ8pggl9FOUGuCLVfe+NKxp2k9sEMCH3tHIEg==}
engines: {node: '>=14'}
peek-readable@5.0.0:
resolution: {integrity: sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==}
engines: {node: '>=14.16'}
pend@1.2.0:
resolution: {integrity: sha1-elfrVQpng/kRUzH89GY9XI4AelA=}
@ -2464,6 +2486,10 @@ packages:
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
engines: {node: '>= 6'}
readable-web-to-node-stream@3.0.2:
resolution: {integrity: sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==}
engines: {node: '>=8'}
readdirp@3.6.0:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
@ -2682,6 +2708,10 @@ packages:
strnum@1.0.5:
resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}
strtok3@7.0.0:
resolution: {integrity: sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==}
engines: {node: '>=14.16'}
stubborn-fs@1.2.5:
resolution: {integrity: sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==}
@ -2732,6 +2762,10 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
token-types@5.0.1:
resolution: {integrity: sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==}
engines: {node: '>=14.16'}
tr46@0.0.3:
resolution: {integrity: sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=}
@ -3630,6 +3664,12 @@ snapshots:
'@types/mime-types': 2.1.4
mime-types: 2.1.35
'@push.rocks/smartmime@2.0.0':
dependencies:
'@types/mime-types': 2.1.4
file-type: 19.0.0
mime: 4.0.3
'@push.rocks/smartnetwork@3.0.2':
dependencies:
'@pushrocks/smartping': 1.0.8
@ -4037,6 +4077,8 @@ snapshots:
dependencies:
stubborn-fs: 1.2.5
'@tokenizer/token@0.3.0': {}
'@tsclass/tsclass@3.0.48':
dependencies:
type-fest: 2.19.0
@ -4969,6 +5011,12 @@ snapshots:
dependencies:
is-unicode-supported: 2.0.0
file-type@19.0.0:
dependencies:
readable-web-to-node-stream: 3.0.2
strtok3: 7.0.0
token-types: 5.0.1
fill-range@7.0.1:
dependencies:
to-regex-range: 5.0.1
@ -5956,6 +6004,8 @@ snapshots:
mime@1.6.0: {}
mime@4.0.3: {}
mimic-fn@2.1.0: {}
mimic-response@3.1.0: {}
@ -6125,6 +6175,8 @@ snapshots:
transitivePeerDependencies:
- supports-color
peek-readable@5.0.0: {}
pend@1.2.0: {}
picocolors@1.0.1: {}
@ -6248,6 +6300,10 @@ snapshots:
string_decoder: 1.3.0
util-deprecate: 1.0.2
readable-web-to-node-stream@3.0.2:
dependencies:
readable-stream: 3.6.2
readdirp@3.6.0:
dependencies:
picomatch: 2.3.1
@ -6530,6 +6586,11 @@ snapshots:
strnum@1.0.5: {}
strtok3@7.0.0:
dependencies:
'@tokenizer/token': 0.3.0
peek-readable: 5.0.0
stubborn-fs@1.2.5: {}
supports-color@5.5.0:
@ -6588,6 +6649,11 @@ snapshots:
toidentifier@1.0.1: {}
token-types@5.0.1:
dependencies:
'@tokenizer/token': 0.3.0
ieee754: 1.2.1
tr46@0.0.3: {}
tr46@2.1.0:

View File

@ -1,5 +1,6 @@
# @push.rocks/smartbucket
A TypeScript library for simple cloud independent object storage with support for buckets, directories, and files.
A TypeScript library that offers simple, cloud-independent object storage with features like bucket creation, file management, and directory management.
## Install
@ -199,12 +200,15 @@ writeFileStream("exampleBucket", "path/to/streamedObject.txt", readable);
`@push.rocks/smartbucket` abstracts directories within buckets for easier object management. You can create, list, and delete directories using the `Directory` class.
Here's how to list the contents of a directory:
```typescript
async function listDirectoryContents(bucketName: string, directoryPath: string) {
const myBucket: Bucket = await mySmartBucket.getBucketByName(bucketName);
if (myBucket) {
const baseDirectory: Directory = await myBucket.getBaseDirectory();
const targetDirectory: Directory = await baseDirectory.getSubDirectoryByName(directoryPath);
console.log('Listing directories:');
const directories = await targetDirectory.listDirectories();
directories.forEach(dir => {
@ -304,6 +308,8 @@ Remember, each cloud provider has specific features and limitations. `@push.rock
This guide covers the basic to advanced scenarios of using `@push.rocks/smartbucket`. For further details, refer to the API documentation and examples.
## License and Legal Information
This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.

View File

@ -79,7 +79,7 @@ tap.test('prepare for directory style tests', async () => {
contents: 'dir3/dir4/file1.txt content',
});
await myBucket.fastPut({
path: 'file1.txt',
path: '/file1.txt',
contents: 'file1 content',
});
});

View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartbucket',
version: '3.0.3',
description: 'A TypeScript library that offers simple, cloud-independent object storage with features like bucket creation, file management, and directory management.'
version: '3.0.8',
description: 'A TypeScript library for cloud-independent object storage, providing features like bucket creation, file and directory management, and data streaming.'
}

368
ts/classes.bucket.ts Normal file
View File

@ -0,0 +1,368 @@
import * as plugins from './plugins.js';
import * as helpers from './helpers.js';
import * as interfaces from './interfaces.js';
import { SmartBucket } from './classes.smartbucket.js';
import { Directory } from './classes.directory.js';
import { File } from './classes.file.js';
export class Bucket {
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
const buckets = await smartbucketRef.minioClient.listBuckets();
const foundBucket = buckets.find((bucket) => {
return bucket.name === bucketNameArg;
});
if (foundBucket) {
console.log(`bucket with name ${bucketNameArg} exists.`);
console.log(`Taking this as base for new Bucket instance`);
return new this(smartbucketRef, bucketNameArg);
} else {
return null;
}
}
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
await smartbucketRef.minioClient.makeBucket(bucketName, 'ams3').catch((e) => console.log(e));
return new Bucket(smartbucketRef, bucketName);
}
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
await smartbucketRef.minioClient.removeBucket(bucketName).catch((e) => console.log(e));
}
public smartbucketRef: SmartBucket;
public name: string;
constructor(smartbucketRef: SmartBucket, bucketName: string) {
this.smartbucketRef = smartbucketRef;
this.name = bucketName;
}
/**
* gets the base directory of the bucket
*/
public async getBaseDirectory(): Promise<Directory> {
return new Directory(this, null, '');
}
public async getDirectoryFromPath(pathDescriptorArg: interfaces.IPathDecriptor): Promise<Directory> {
if (!pathDescriptorArg.path && !pathDescriptorArg.directory) {
return this.getBaseDirectory();
}
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptorArg);
const baseDirectory = await this.getBaseDirectory();
return await baseDirectory.getSubDirectoryByName(checkPath);
}
// ===============
// Fast Operations
// ===============
/**
* store file
*/
public async fastPut(optionsArg: {
path: string;
contents: string | Buffer;
overwrite?: boolean;
}): Promise<File> {
try {
const reducedPath = await helpers.reducePathDescriptorToPath({
path: optionsArg.path,
})
// Check if the object already exists
const exists = await this.fastExists({ path: reducedPath });
if (exists && !optionsArg.overwrite) {
console.error(`Object already exists at path '${reducedPath}' in bucket '${this.name}'.`);
return;
} else if (exists && optionsArg.overwrite) {
console.log(`Overwriting existing object at path '${reducedPath}' in bucket '${this.name}'.`);
} else {
console.log(`Creating new object at path '${reducedPath}' in bucket '${this.name}'.`);
}
// Proceed with putting the object
const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = this.smartbucketRef.minioClient.putObject(this.name, reducedPath, streamIntake);
streamIntake.pushData(optionsArg.contents);
streamIntake.signalEnd();
await putPromise;
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
const parsedPath = plugins.path.parse(reducedPath);
return new File({
directoryRefArg: await this.getDirectoryFromPath({
path: parsedPath.dir,
}),
fileName: parsedPath.base,
});
} catch (error) {
console.error(`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`, error);
throw error;
}
}
/**
* get file
*/
public async fastGet(optionsArg: Parameters<typeof this.fastGetStream>[0]): Promise<Buffer> {
const done = plugins.smartpromise.defer();
let completeFile: Buffer;
const replaySubject = await this.fastGetStream(optionsArg);
const subscription = replaySubject.subscribe({
next: (chunk) => {
if (completeFile) {
completeFile = Buffer.concat([completeFile, chunk]);
} else {
completeFile = chunk;
}
},
complete: () => {
done.resolve();
subscription.unsubscribe();
},
error: (err) => {
console.log(err);
},
});
await done.promise;
return completeFile;
}
public async fastGetStream(optionsArg: {
path: string;
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const fileStream = await this.smartbucketRef.minioClient
.getObject(this.name, optionsArg.path)
.catch((e) => console.log(e));
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, void>({
writeFunction: async (chunk) => {
replaySubject.next(chunk);
return;
},
finalFunction: async (cb) => {
replaySubject.complete();
return;
}
});
if (!fileStream) {
return null;
}
const smartstream = new plugins.smartstream.StreamWrapper([
fileStream,
duplexStream,
]);
smartstream.run();
return replaySubject;
}
/**
* store file as stream
*/
public async fastPutStream(optionsArg: {
path: string;
dataStream: plugins.stream.Readable;
nativeMetadata?: { [key: string]: string };
overwrite?: boolean;
}): Promise<void> {
try {
// Check if the object already exists
const exists = await this.fastExists({ path: optionsArg.path });
if (exists && !optionsArg.overwrite) {
console.error(`Object already exists at path '${optionsArg.path}' in bucket '${this.name}'.`);
return;
} else if (exists && optionsArg.overwrite) {
console.log(`Overwriting existing object at path '${optionsArg.path}' in bucket '${this.name}'.`);
} else {
console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`);
}
// Proceed with putting the object
await this.smartbucketRef.minioClient.putObject(
this.name,
optionsArg.path,
optionsArg.dataStream,
null,
...(optionsArg.nativeMetadata
? (() => {
const returnObject: any = {};
return returnObject;
})()
: {})
);
console.log(`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`);
} catch (error) {
console.error(`Error storing object at path '${optionsArg.path}' in bucket '${this.name}':`, error);
throw error;
}
}
public async fastCopy(optionsArg: {
sourcePath: string;
destinationPath?: string;
targetBucket?: Bucket;
nativeMetadata?: { [key: string]: string };
deleteExistingNativeMetadata?: boolean;
}): Promise<void> {
try {
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
// Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
targetBucketName,
optionsArg.sourcePath
);
// Setting up copy conditions
const copyConditions = new plugins.minio.CopyConditions();
// Prepare new metadata
const newNativeMetadata = {
...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.metaData),
...optionsArg.nativeMetadata,
};
// Define the copy operation as a Promise
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
await this.smartbucketRef.minioClient.copyObject(
this.name,
optionsArg.sourcePath,
`/${targetBucketName}/${optionsArg.destinationPath || optionsArg.sourcePath}`,
copyConditions
);
} catch (err) {
console.error('Error updating metadata:', err);
throw err; // rethrow to allow caller to handle
}
}
/**
* Move object from one path to another within the same bucket or to another bucket
*/
public async fastMove(optionsArg: {
sourcePath: string;
destinationPath: string;
targetBucket?: Bucket;
overwrite?: boolean;
}): Promise<void> {
try {
// Check if the destination object already exists
const destinationBucket = optionsArg.targetBucket || this;
const exists = await destinationBucket.fastExists({ path: optionsArg.destinationPath });
if (exists && !optionsArg.overwrite) {
console.error(`Object already exists at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
return;
} else if (exists && optionsArg.overwrite) {
console.log(`Overwriting existing object at destination path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
} else {
console.log(`Moving object to path '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
}
// Proceed with copying the object to the new path
await this.fastCopy(optionsArg);
// Remove the original object after successful copy
await this.fastRemove({ path: optionsArg.sourcePath });
console.log(`Object '${optionsArg.sourcePath}' has been successfully moved to '${optionsArg.destinationPath}' in bucket '${destinationBucket.name}'.`);
} catch (error) {
console.error(`Error moving object from '${optionsArg.sourcePath}' to '${optionsArg.destinationPath}':`, error);
throw error;
}
}
/**
* removeObject
*/
public async fastRemove(optionsArg: {
path: string;
}) {
await this.smartbucketRef.minioClient.removeObject(this.name, optionsArg.path);
}
/**
* check wether file exists
* @param optionsArg
* @returns
*/
public async fastExists(optionsArg: {
path: string;
}): Promise<boolean> {
try {
await this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path);
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
return true;
} catch (error) {
if (error.code === 'NotFound') {
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
return false;
} else {
console.error('Error checking object existence:', error);
throw error; // Rethrow if it's not a NotFound error to handle unexpected issues
}
}
}
public async fastStat(pathDescriptor: interfaces.IPathDecriptor) {
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
return this.smartbucketRef.minioClient.statObject(this.name, checkPath);
}
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
// lets check if the checkPath is a directory
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
const done = plugins.smartpromise.defer<boolean>();
stream.on('data', (dataArg) => {
stream.destroy(); // Stop the stream early if we find at least one object
if (dataArg.prefix.startsWith(checkPath + '/')) {
done.resolve(true);
}
});
stream.on('end', () => {
done.resolve(false);
});
stream.on('error', (err) => {
done.reject(err);
});
return done.promise;
};
public async isFile(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor);
// lets check if the checkPath is a directory
const stream = this.smartbucketRef.minioClient.listObjectsV2(this.name, checkPath, true);
const done = plugins.smartpromise.defer<boolean>();
stream.on('data', (dataArg) => {
stream.destroy(); // Stop the stream early if we find at least one object
if (dataArg.prefix === checkPath) {
done.resolve(true);
}
});
stream.on('end', () => {
done.resolve(false);
});
stream.on('error', (err) => {
done.reject(err);
});
return done.promise;
}
}

View File

@ -1,6 +1,6 @@
import * as plugins from './smartbucket.plugins.js';
import { Bucket } from './smartbucket.classes.bucket.js';
import { File } from './smartbucket.classes.file.js';
import * as plugins from './plugins.js';
import { Bucket } from './classes.bucket.js';
import { File } from './classes.file.js';
export class Directory {
public bucketRef: Bucket;
@ -59,6 +59,32 @@ export class Directory {
return basePath;
}
/**
* gets a file by name
*/
public async getFile(optionsArg: {
name: string;
createWithContents?: string | Buffer;
}): Promise<File> {
// check wether the file exists
const exists = await this.bucketRef.fastExists({
path: this.getBasePath() + optionsArg.name,
});
if (!exists && !optionsArg.createWithContents) {
return null;
}
if (!exists && optionsArg.createWithContents) {
await this.fastPut({
path: optionsArg.name,
contents: optionsArg.createWithContents,
});
}
return new File({
directoryRefArg: this,
fileName: optionsArg.name,
})
}
/**
* lists all files
*/

200
ts/classes.file.ts Normal file
View File

@ -0,0 +1,200 @@
import * as plugins from './plugins.js';
import * as helpers from './helpers.js';
import * as interfaces from './interfaces.js';
import { Directory } from './classes.directory.js';
import { MetaData } from './classes.metadata.js';
/**
* represents a file in a directory
*/
export class File {
// STATIC
/**
* creates a file in draft mode
* you need to call .save() to store it in s3
* @param optionsArg
*/
public static async create(optionsArg: {
directory: Directory;
name: string;
contents: Buffer | string | plugins.stream.Readable;
/**
* if contents are of type string, you can specify the encoding here
*/
encoding?: 'utf8' | 'binary';
}): Promise<File> {
const contents =
typeof optionsArg.contents === 'string'
? Buffer.from(optionsArg.contents, optionsArg.encoding)
: optionsArg.contents;
const file = new File({
directoryRefArg: optionsArg.directory,
fileName: optionsArg.name,
});
if (contents instanceof plugins.stream.Readable) {} else {
await optionsArg.directory.fastPut({
path: optionsArg.name,
contents: contents,
});
}
return file;
}
// INSTANCE
public parentDirectoryRef: Directory;
public name: string;
public getBasePath(): string {
return plugins.path.join(this.parentDirectoryRef.getBasePath(), this.name);
};
constructor(optionsArg: { directoryRefArg: Directory; fileName: string }) {
this.parentDirectoryRef = optionsArg.directoryRefArg;
this.name = optionsArg.fileName;
}
public async getContentsAsString(): Promise<string> {
const fileBuffer = await this.getContents();
return fileBuffer.toString();
}
public async getContents(): Promise<Buffer> {
const resultBuffer = await this.parentDirectoryRef.bucketRef.fastGet({
path: this.getBasePath(),
});
return resultBuffer;
}
public async getReadStream() {
const readStream = this.parentDirectoryRef.bucketRef.fastGetStream({
path: this.getBasePath(),
});
}
/**
* removes this file
* for using recycling mechanics use .delete()
*/
public async remove() {
await this.parentDirectoryRef.bucketRef.fastRemove({
path: this.getBasePath(),
});
if (!this.name.endsWith('.metadata')) {
await this.parentDirectoryRef.bucketRef.fastRemove({
path: this.getBasePath() + '.metadata',
});
}
await this.parentDirectoryRef.listFiles();
}
/**
* deletes the file with recycling mechanics
*/
public async delete() {
await this.remove();
}
/**
* allows locking the file
* @param optionsArg
*/
public async lock(optionsArg?: { timeoutMillis?: number }) {
const metadata = await this.getMetaData();
await metadata.setLock({
lock: 'locked',
expires: Date.now() + (optionsArg?.timeoutMillis || 1000),
});
}
/**
* actively unlocks a file
*
*/
public async unlock(optionsArg?: {
/**
* unlock the file even if not locked from this instance
*/
force?: boolean;
}) {
const metadata = await this.getMetaData();
await metadata.removeLock({
force: optionsArg?.force,
});
}
public async updateWithContents(optionsArg: {
contents: Buffer | string | plugins.stream.Readable;
encoding?: 'utf8' | 'binary';
}) {
if (optionsArg.contents instanceof plugins.stream.Readable) {
await this.parentDirectoryRef.bucketRef.fastPutStream({
path: this.getBasePath(),
dataStream: optionsArg.contents,
});
} else if (Buffer.isBuffer(optionsArg.contents)) {
await this.parentDirectoryRef.bucketRef.fastPut({
path: this.getBasePath(),
contents: optionsArg.contents,
});
} else if (typeof optionsArg.contents === 'string') {
await this.parentDirectoryRef.bucketRef.fastPut({
path: this.getBasePath(),
contents: Buffer.from(optionsArg.contents, optionsArg.encoding),
});
}
}
/**
* moves the file to another directory
*/
public async move(pathDescriptorArg: interfaces.IPathDecriptor) {
let moveToPath = '';
const isDirectory = await this.parentDirectoryRef.bucketRef.isDirectory(pathDescriptorArg);
if (isDirectory) {
moveToPath = await helpers.reducePathDescriptorToPath({
...pathDescriptorArg,
path: plugins.path.join(pathDescriptorArg.path, this.name),
});
}
// lets move the file
await this.parentDirectoryRef.bucketRef.fastMove({
sourcePath: this.getBasePath(),
destinationPath: moveToPath,
});
// lets move the metadatafile
const metadata = await this.getMetaData();
await metadata.metadataFile.move(pathDescriptorArg);
}
/**
* allows updating the metadata of a file
* @param updatedMetadata
*/
public async getMetaData() {
if (this.name.endsWith('.metadata')) {
throw new Error('metadata files cannot have metadata');
}
const metadata = await MetaData.createForFile({
file: this,
});
return metadata;
}
/**
* gets the contents as json
*/
public async getJsonData() {
const json = await this.getContentsAsString();
const parsed = await JSON.parse(json);
return parsed;
}
public async writeJsonData(dataArg: any) {
await this.updateWithContents({
contents: JSON.stringify(dataArg),
});
}
}

100
ts/classes.metadata.ts Normal file
View File

@ -0,0 +1,100 @@
import * as plugins from './plugins.js';
import { File } from './classes.file.js';
export class MetaData {
// static
public static async createForFile(optionsArg: { file: File }) {
const metaData = new MetaData();
metaData.fileRef = optionsArg.file;
// lets find the existing metadata file
metaData.metadataFile = await metaData.fileRef.parentDirectoryRef.getFile({
name: metaData.fileRef.name + '.metadata',
createWithContents: '{}',
});
return metaData;
}
// instance
/**
* the file that contains the metadata
*/
metadataFile: File;
/**
* the file that the metadata is for
*/
fileRef: File;
public async getFileType(optionsArg?: {
useFileExtension?: boolean;
useMagicBytes?: boolean;
}): Promise<string> {
if ((optionsArg && optionsArg.useFileExtension) || optionsArg.useFileExtension === undefined) {
return plugins.path.extname(this.fileRef.name);
}
}
/**
* gets the size of the fileRef
*/
public async getSizeInBytes(): Promise<number> {
const stat = await this.fileRef.parentDirectoryRef.bucketRef.fastStat({
path: this.fileRef.getBasePath(),
});
return stat.size;
}
private prefixCustomMetaData = 'custom_';
public async storeCustomMetaData<T = any>(optionsArg: { key: string; value: T }) {
const data = await this.metadataFile.getContentsAsString();
data[this.prefixCustomMetaData + optionsArg.key] = optionsArg.value;
await this.metadataFile.writeJsonData(data);
}
public async getCustomMetaData<T = any>(optionsArg: { key: string }): Promise<T> {
const data = await this.metadataFile.getJsonData();
return data[this.prefixCustomMetaData + optionsArg.key];
}
public async deleteCustomMetaData(optionsArg: { key: string }) {
const data = await this.metadataFile.getJsonData();
delete data[this.prefixCustomMetaData + optionsArg.key];
await this.metadataFile.writeJsonData(data);
}
/**
* set a lock on the ref file
* @param optionsArg
*/
public async setLock(optionsArg: { lock: string; expires: number }) {
const data = await this.metadataFile.getJsonData();
data.lock = optionsArg.lock;
data.lockExpires = optionsArg.expires;
await this.metadataFile.writeJsonData(data);
}
/**
* remove the lock on the ref file
* @param optionsArg
*/
public async removeLock(optionsArg: { force: boolean }) {
const data = await this.metadataFile.getJsonData();
delete data.lock;
delete data.lockExpires;
await this.metadataFile.writeJsonData(data);
}
public async checkLocked(): Promise<boolean> {
const data = await this.metadataFile.getJsonData();
return data.lock && data.lockExpires > Date.now();
}
public async getLockInfo(): Promise<{ lock: string; expires: number }> {
const data = await this.metadataFile.getJsonData();
return { lock: data.lock, expires: data.lockExpires };
}
}

View File

@ -1,5 +1,5 @@
import * as plugins from './smartbucket.plugins.js';
import { Bucket } from './smartbucket.classes.bucket.js';
import * as plugins from './plugins.js';
import { Bucket } from './classes.bucket.js';
export class SmartBucket {
public config: plugins.tsclass.storage.IS3Descriptor;

22
ts/helpers.ts Normal file
View File

@ -0,0 +1,22 @@
import * as plugins from './plugins.js';
import * as interfaces from './interfaces.js';
export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.IPathDecriptor): Promise<string> => {
let returnPath = ``
if (pathDescriptorArg.directory) {
if (pathDescriptorArg.path && plugins.path.isAbsolute(pathDescriptorArg.path)) {
console.warn('Directory is being ignored when path is absolute.');
returnPath = pathDescriptorArg.path;
} else if (pathDescriptorArg.path) {
returnPath = plugins.path.join(pathDescriptorArg.directory.getBasePath(), pathDescriptorArg.path);
}
} else if (pathDescriptorArg.path) {
returnPath = pathDescriptorArg.path;
} else {
throw new Error('You must specify either a path or a directory.');
}
if (returnPath.startsWith('/')) {
returnPath = returnPath.substring(1);
}
return returnPath;
}

View File

@ -1,4 +1,4 @@
export * from './smartbucket.classes.smartbucket.js';
export * from './smartbucket.classes.bucket.js';
export * from './smartbucket.classes.directory.js';
export * from './smartbucket.classes.file.js';
export * from './classes.smartbucket.js';
export * from './classes.bucket.js';
export * from './classes.directory.js';
export * from './classes.file.js';

6
ts/interfaces.ts Normal file
View File

@ -0,0 +1,6 @@
import type { Directory } from "./classes.directory.js";
export interface IPathDecriptor {
path?: string;
directory?: Directory;
}

View File

@ -5,12 +5,13 @@ import * as stream from 'stream';
export { path, stream };
// @push.rocks scope
import * as smartmime from '@push.rocks/smartmime';
import * as smartpath from '@push.rocks/smartpath';
import * as smartpromise from '@push.rocks/smartpromise';
import * as smartrx from '@push.rocks/smartrx';
import * as smartstream from '@push.rocks/smartstream';
export { smartpath, smartpromise, smartrx, smartstream };
export { smartmime, smartpath, smartpromise, smartrx, smartstream };
// @tsclass
import * as tsclass from '@tsclass/tsclass';

View File

@ -1,218 +0,0 @@
import * as plugins from './smartbucket.plugins.js';
import { SmartBucket } from './smartbucket.classes.smartbucket.js';
import { Directory } from './smartbucket.classes.directory.js';
export class Bucket {
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string) {
const buckets = await smartbucketRef.minioClient.listBuckets();
const foundBucket = buckets.find((bucket) => {
return bucket.name === bucketNameArg;
});
if (foundBucket) {
console.log(`bucket with name ${bucketNameArg} exists.`);
console.log(`Taking this as base for new Bucket instance`);
return new this(smartbucketRef, bucketNameArg);
} else {
return null;
}
}
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
await smartbucketRef.minioClient.makeBucket(bucketName, 'ams3').catch((e) => console.log(e));
return new Bucket(smartbucketRef, bucketName);
}
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
await smartbucketRef.minioClient.removeBucket(bucketName).catch((e) => console.log(e));
}
public smartbucketRef: SmartBucket;
public name: string;
constructor(smartbucketRef: SmartBucket, bucketName: string) {
this.smartbucketRef = smartbucketRef;
this.name = bucketName;
}
/**
* gets the base directory of the bucket
*/
public async getBaseDirectory() {
return new Directory(this, null, '');
}
// ===============
// Fast Operations
// ===============
/**
* store file
*/
public async fastPut(optionsArg: {
path: string;
contents: string | Buffer;
}): Promise<void> {
const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = this.smartbucketRef.minioClient
.putObject(this.name, optionsArg.path, streamIntake)
.catch((e) => console.log(e));
streamIntake.pushData(optionsArg.contents);
streamIntake.signalEnd();
const response = await putPromise;
}
/**
* get file
*/
public async fastGet(optionsArg: Parameters<typeof this.fastGetStream>[0]): Promise<Buffer> {
const done = plugins.smartpromise.defer();
let completeFile: Buffer;
const replaySubject = await this.fastGetStream(optionsArg);
const subscription = replaySubject.subscribe({
next: (chunk) => {
if (completeFile) {
completeFile = Buffer.concat([completeFile, chunk]);
} else {
completeFile = chunk;
}
},
complete: () => {
done.resolve();
subscription.unsubscribe();
},
error: (err) => {
console.log(err);
},
});
await done.promise;
return completeFile;
}
public async fastGetStream(optionsArg: {
path: string;
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const fileStream = await this.smartbucketRef.minioClient
.getObject(this.name, optionsArg.path)
.catch((e) => console.log(e));
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, void>({
writeFunction: async (chunk) => {
replaySubject.next(chunk);
return;
},
finalFunction: async (cb) => {
replaySubject.complete();
return;
}
});
if (!fileStream) {
return null;
}
const smartstream = new plugins.smartstream.StreamWrapper([
fileStream,
duplexStream,
]);
smartstream.run();
return replaySubject;
}
/**
* store file as stream
*/
public async fastPutStream(optionsArg: {
path: string;
dataStream: plugins.stream.Readable;
nativeMetadata?: { [key: string]: string };
}): Promise<void> {
await this.smartbucketRef.minioClient.putObject(
this.name,
optionsArg.path,
optionsArg.dataStream,
null,
...(optionsArg.nativeMetadata
? (() => {
const returnObject: any = {};
return returnObject;
})()
: {})
);
}
public async copyObject(optionsArg: {
/**
* the
*/
objectKey: string;
/**
* in case you want to copy to another bucket specify it here
*/
targetBucket?: Bucket;
targetBucketKey?: string;
/**
* metadata will be merged with existing metadata
*/
nativeMetadata?: { [key: string]: string };
deleteExistingNativeMetadata?: boolean;
}): Promise<void> {
try {
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
// Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
targetBucketName,
optionsArg.objectKey
);
// Setting up copy conditions
const copyConditions = new plugins.minio.CopyConditions();
// Prepare new metadata
const newNativeMetadata = {
...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.metaData),
...optionsArg.nativeMetadata,
};
// Define the copy operation as a Promise
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
await this.smartbucketRef.minioClient.copyObject(
this.name,
optionsArg.objectKey,
`/${targetBucketName}/${optionsArg.objectKey}`,
copyConditions
);
} catch (err) {
console.error('Error updating metadata:', err);
throw err; // rethrow to allow caller to handle
}
}
/**
* removeObject
*/
public async fastRemove(optionsArg: {
path: string;
}) {
await this.smartbucketRef.minioClient.removeObject(this.name, optionsArg.path);
}
public async doesObjectExist(optionsArg: {
path: string;
}): Promise<boolean> {
try {
await this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path);
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
return true;
} catch (error) {
if (error.code === 'NotFound') {
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
return false;
} else {
console.error('Error checking object existence:', error);
throw error; // Rethrow if it's not a NotFound error to handle unexpected issues
}
}
}
}

View File

@ -1,140 +0,0 @@
import * as plugins from './smartbucket.plugins.js';
import { Directory } from './smartbucket.classes.directory.js';
export interface IFileMetaData {
name: string;
fileType: string;
size: string;
}
/**
* represents a file in a directory
*/
export class File {
// STATIC
/**
* creates a file in draft mode
* you need to call .save() to store it in s3
* @param optionsArg
*/
public static async create(optionsArg: {
directory: Directory;
name: string;
contents: Buffer | string | plugins.stream.Readable;
/**
* if contents are of type string, you can specify the encoding here
*/
encoding?: 'utf8' | 'binary';
}): Promise<File> {
const contents =
typeof optionsArg.contents === 'string'
? Buffer.from(optionsArg.contents, optionsArg.encoding)
: optionsArg.contents;
const file = new File({
directoryRefArg: optionsArg.directory,
fileName: optionsArg.name,
});
if (contents instanceof plugins.stream.Readable) {} else {
await optionsArg.directory.fastPut({
path: optionsArg.name,
contents: contents,
});
}
return file;
}
// INSTANCE
public parentDirectoryRef: Directory;
public name: string;
public path: string;
public metaData: IFileMetaData;
constructor(optionsArg: { directoryRefArg: Directory; fileName: string }) {
this.parentDirectoryRef = optionsArg.directoryRefArg;
this.name = optionsArg.fileName;
}
public async getContentAsString() {
const fileBuffer = await this.getContentAsBuffer();
return fileBuffer.toString();
}
public async getContentAsBuffer() {
const done = plugins.smartpromise.defer();
const fileStream = await this.parentDirectoryRef.bucketRef.smartbucketRef.minioClient
.getObject(this.parentDirectoryRef.bucketRef.name, this.path)
.catch((e) => console.log(e));
let completeFile = Buffer.from('');
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>(
{
writeFunction: async (chunk) => {
completeFile = Buffer.concat([chunk]);
return chunk;
},
finalFunction: async (cb) => {
done.resolve();
return Buffer.from('');
},
}
);
if (!fileStream) {
return null;
}
fileStream.pipe(duplexStream);
await done.promise;
return completeFile;
}
public async readStreaming() {
// TODO
throw new Error('not yet implemented');
}
/**
* removes this file
*/
public async remove() {
await this.parentDirectoryRef.bucketRef.smartbucketRef.minioClient.removeObject(
this.parentDirectoryRef.bucketRef.name,
this.path
);
await this.parentDirectoryRef.listFiles();
}
/**
* deletes the file
*/
public async delete() {}
/**
* allows locking the file
* @param optionsArg
*/
public async lock(optionsArg?: { timeoutMillis?: number }) {}
/**
* actively unlocks a file
*
*/
public async unlock(optionsArg?: {
/**
* unlock the file even if not locked from this instance
*/
force?: boolean;
}) {}
public async updateWithContents(optionsArg: {
contents: Buffer | string | plugins.stream.Readable;
encoding?: 'utf8' | 'binary';
}) {}
/**
* allows updating the metadata of a file
* @param updatedMetadata
*/
public async updateMetaData(updatedMetadata: any) {}
}