push.rocks/smartbucket
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(core): update

Browse Source
This commit is contained in:
Philipp Kunz 2024-05-17 18:53:11 +02:00
parent dab74572b8
commit a0be96bf23
8 changed files with 13609 additions and 14048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17893
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

17
package.json
View File

@ -12,20 +12,19 @@
"build": "(tsbuild --web --allowimplicitany)"
},
"devDependencies": {
"@git.zone/tsbuild": "^2.1.63",
"@git.zone/tsbuild": "^2.1.76",
"@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.71",
"@push.rocks/qenv": "^6.0.4",
"@push.rocks/tapbundle": "^5.0.3"
"@git.zone/tstest": "^1.0.90",
"@push.rocks/qenv": "^6.0.5",
"@push.rocks/tapbundle": "^5.0.23"
},
"dependencies": {
"@push.rocks/smartpath": "^5.0.5",
"@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrx": "^3.0.7",
"@push.rocks/smartstream": "^2.0.2",
"@tsclass/tsclass": "^4.0.50",
"@types/minio": "^7.0.13",
"minio": "^7.0.28"
"@push.rocks/smartstream": "^3.0.37",
"@tsclass/tsclass": "^4.0.54",
"minio": "^8.0.0"
},
"private": false,
"files": [

9332
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

65
test/test.ts
View File

@ -33,28 +33,55 @@ tap.test('should get a bucket', async () => {
// Fast operations
tap.test('should store data in bucket fast', async () => {
await myBucket.fastPut('hithere/socool.txt', 'hi there!');
await myBucket.fastPut({
path: 'hithere/socool.txt',
contents: 'hi there!',
});
});
tap.test('should get data in bucket', async () => {
const fileString = await myBucket.fastGet('hithere/socool.txt');
const fileStringStream = await myBucket.fastGetStream('hithere/socool.txt');
const fileString = await myBucket.fastGet({
path: 'hithere/socool.txt',
});
const fileStringStream = await myBucket.fastGetStream({
path: 'hithere/socool.txt',
});
console.log(fileString);
});
tap.test('should delete data in bucket', async () => {
await myBucket.fastRemove('hithere/socool.txt');
await myBucket.fastRemove({
path: 'hithere/socool.txt',
});
});
// fs operations
tap.test('prepare for directory style tests', async () => {
await myBucket.fastPut('dir1/file1.txt', 'dir1/file1.txt content');
await myBucket.fastPut('dir1/file2.txt', 'dir1/file2.txt content');
await myBucket.fastPut('dir2/file1.txt', 'dir2/file1.txt content');
await myBucket.fastPut('dir3/file1.txt', 'dir3/file1.txt content');
await myBucket.fastPut('dir3/dir4/file1.txt', 'dir3/dir4/file1.txt content');
await myBucket.fastPut('file1.txt', 'file1 content');
await myBucket.fastPut({
path: 'dir1/file1.txt',
contents: 'dir1/file1.txt content',
});
await myBucket.fastPut({
path: 'dir1/file2.txt',
contents: 'dir1/file2.txt content',
});
await myBucket.fastPut({
path: 'dir2/file1.txt',
contents: 'dir2/file1.txt content',
});
await myBucket.fastPut({
path: 'dir3/file1.txt',
contents: 'dir3/file1.txt content',
});
await myBucket.fastPut({
path: 'dir3/dir4/file1.txt',
contents: 'dir3/dir4/file1.txt content',
});
await myBucket.fastPut({
path: 'file1.txt',
contents: 'file1 content',
});
});
tap.test('should get base directory', async () => {
@ -77,12 +104,18 @@ tap.test('should correctly build paths for sub directories', async () => {
});
tap.test('clean up directory style tests', async () => {
await myBucket.fastRemove('dir1/file1.txt');
await myBucket.fastRemove('dir1/file2.txt');
await myBucket.fastRemove('dir2/file1.txt');
await myBucket.fastRemove('dir3/file1.txt');
await myBucket.fastRemove('dir3/dir4/file1.txt');
await myBucket.fastRemove('file1.txt');
await myBucket.fastRemove({
path: 'dir1/file1.txt',
});
await myBucket.fastRemove({
path: 'dir1/file2.txt',
});
await myBucket.fastRemove({
path: 'dir2/file1.txt',
});
await myBucket.fastRemove({ path: 'dir3/file1.txt' });
await myBucket.fastRemove({ path: 'dir3/dir4/file1.txt' });
await myBucket.fastRemove({ path: 'file1.txt' });
});
tap.start();

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/
export const commitinfo = {
name: '@push.rocks/smartbucket',
version: '2.0.5',
version: '3.0.0',
description: 'A TypeScript library for simple cloud independent object storage with support for buckets, directories, and files.'
}

123
ts/smartbucket.classes.bucket.ts
View File

@ -49,58 +49,63 @@ export class Bucket {
/**
* store file
*/
public async fastPut(pathArg: string, fileContent: string | Buffer): Promise<void> {
public async fastPut(optionsArg: {
path: string;
contents: string | Buffer;
}): Promise<void> {
const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = this.smartbucketRef.minioClient
.putObject(this.name, pathArg, streamIntake.getReadable())
.putObject(this.name, optionsArg.path, streamIntake)
.catch((e) => console.log(e));
streamIntake.pushData(fileContent);
streamIntake.pushData(optionsArg.contents);
streamIntake.signalEnd();
await putPromise;
const response = await putPromise;
}
/**
* get file
*/
public async fastGet(pathArg: string): Promise<Buffer> {
public async fastGet(optionsArg: Parameters<typeof this.fastGetStream>[0]): Promise<Buffer> {
const done = plugins.smartpromise.defer();
let completeFile: Buffer;
const replaySubject = await this.fastGetStream(pathArg);
const subscription = replaySubject.subscribe(
(chunk) => {
const replaySubject = await this.fastGetStream(optionsArg);
const subscription = replaySubject.subscribe({
next: (chunk) => {
if (completeFile) {
completeFile = Buffer.concat([completeFile, chunk]);
} else {
completeFile = chunk;
}
},
(err) => {
console.log(err);
},
() => {
complete: () => {
done.resolve();
subscription.unsubscribe();
}
);
},
error: (err) => {
console.log(err);
},
});
await done.promise;
return completeFile;
}
public async fastGetStream(pathArg: string): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
public async fastGetStream(optionsArg: {
path: string;
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const fileStream = await this.smartbucketRef.minioClient
.getObject(this.name, pathArg)
.getObject(this.name, optionsArg.path)
.catch((e) => console.log(e));
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
const duplexStream = plugins.smartstream.createDuplexStream<Buffer, Buffer>(
async (chunk) => {
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
writeFunction: async (chunk) => {
replaySubject.next(chunk);
return chunk;
},
async (cb) => {
finalFunction: async (cb) => {
replaySubject.complete();
return Buffer.from('');
}
);
});
if (!fileStream) {
return null;
@ -109,7 +114,6 @@ export class Bucket {
const smartstream = new plugins.smartstream.StreamWrapper([
fileStream,
duplexStream,
plugins.smartstream.cleanPipe(),
]);
smartstream.run();
return replaySubject;
@ -119,16 +123,16 @@ export class Bucket {
* store file as stream
*/
public async fastPutStream(optionsArg: {
pathArg: string;
path: string;
dataStream: plugins.stream.Readable;
metadata?: { [key: string]: string };
nativeMetadata?: { [key: string]: string };
}): Promise<void> {
await this.smartbucketRef.minioClient.putObject(
this.name,
optionsArg.pathArg,
optionsArg.path,
optionsArg.dataStream,
null,
...(optionsArg.metadata
...(optionsArg.nativeMetadata
? (() => {
const returnObject: any = {};
return returnObject;
@ -137,30 +141,47 @@ export class Bucket {
);
}
public async updateMetadata(
bucket: string,
objectKey: string,
metadata: { [key: string]: string }
): Promise<void> {
public async copyObject(optionsArg: {
/**
* the
*/
objectKey: string;
/**
* in case you want to copy to another bucket specify it here
*/
targetBucket?: Bucket;
targetBucketKey?: string;
/**
* metadata will be merged with existing metadata
*/
nativeMetadata?: { [key: string]: string };
deleteExistingNativeMetadata?: boolean;
}): Promise<void> {
try {
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
// Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(bucket, objectKey);
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
targetBucketName,
optionsArg.objectKey
);
// Setting up copy conditions
const copyConditions = new plugins.minio.CopyConditions();
// Prepare new metadata, merging current and new metadata
const newMetadata = {
...currentObjInfo.metaData,
...metadata,
// Prepare new metadata
const newNativeMetadata = {
...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.metaData),
...optionsArg.nativeMetadata,
};
// Define the copy operation as a Promise
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
await this.smartbucketRef.minioClient.copyObject(
bucket,
objectKey,
`/${bucket}/${objectKey}`,
copyConditions,
this.name,
optionsArg.objectKey,
`/${targetBucketName}/${optionsArg.objectKey}`,
copyConditions
);
} catch (err) {
console.error('Error updating metadata:', err);
@ -171,7 +192,27 @@ export class Bucket {
/**
* removeObject
*/
public async fastRemove(pathArg: string) {
await this.smartbucketRef.minioClient.removeObject(this.name, pathArg);
public async fastRemove(optionsArg: {
path: string;
}) {
await this.smartbucketRef.minioClient.removeObject(this.name, optionsArg.path);
}
public async doesObjectExist(optionsArg: {
path: string;
}): Promise<boolean> {
try {
await this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path);
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
return true;
} catch (error) {
if (error.code === 'NotFound') {
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
return false;
} else {
console.error('Error checking object existence:', error);
throw error; // Rethrow if it's not a NotFound error to handle unexpected issues
}
}
}
}

64
ts/smartbucket.classes.directory.ts
View File

@ -70,8 +70,8 @@ export class Directory {
false
);
const fileArray: File[] = [];
const duplexStream = plugins.smartstream.createDuplexStream<plugins.minio.BucketItem, void>(
async (bucketItem) => {
const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
writeFunction: async (bucketItem) => {
if (bucketItem.prefix) {
return;
}
@ -83,13 +83,18 @@ export class Directory {
subtractedPath = subtractedPath.substr(1);
}
if (!subtractedPath.includes('/')) {
fileArray.push(new File(this, subtractedPath));
fileArray.push(
new File({
directoryRefArg: this,
fileName: subtractedPath,
})
);
}
},
async (tools) => {
finalFunction: async (tools) => {
done.resolve();
}
);
});
fileNameStream.pipe(duplexStream);
await done.promise;
return fileArray;
@ -107,8 +112,8 @@ export class Directory {
false
);
const directoryArray: Directory[] = [];
const duplexStream = plugins.smartstream.createDuplexStream<plugins.minio.BucketItem, void>(
async (bucketItem) => {
const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
writeFunction: async (bucketItem) => {
if (bucketItem.name) {
return;
}
@ -124,10 +129,10 @@ export class Directory {
directoryArray.push(new Directory(this.bucketRef, this, dirName));
}
},
async (tools) => {
finalFunction: async (tools) => {
done.resolve();
}
);
});
completeDirStream.pipe(duplexStream);
await done.promise;
return directoryArray;
@ -177,36 +182,49 @@ export class Directory {
* @param relativePathArg
*/
public async createEmptyFile(relativePathArg: string) {
const emtpyFile = await File.createFileFromString(this, relativePathArg, '');
const emtpyFile = await File.create({
directory: this,
name: relativePathArg,
contents: '',
});
}
// file operations
public async fastStore(pathArg: string, contentArg: string | Buffer) {
const path = plugins.path.join(this.getBasePath(), pathArg);
await this.bucketRef.fastPut(path, contentArg);
public async fastPut(optionsArg: { path: string; contents: string | Buffer }) {
const path = plugins.path.join(this.getBasePath(), optionsArg.path);
await this.bucketRef.fastPut({
path,
contents: optionsArg.contents,
});
}
public async fastGet(pathArg: string) {
const path = plugins.path.join(this.getBasePath(), pathArg);
const result = await this.bucketRef.fastGet(path);
public async fastGet(optionsArg: { path: string }) {
const path = plugins.path.join(this.getBasePath(), optionsArg.path);
const result = await this.bucketRef.fastGet({
path,
});
return result;
}
public async fastGetStream(pathArg: string): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const path = plugins.path.join(this.getBasePath(), pathArg);
const result = await this.bucketRef.fastGetStream(path);
const result = await this.bucketRef.fastGetStream({
path,
});
return result;
}
public async fastRemove(pathArg: string) {
const path = plugins.path.join(this.getBasePath(), pathArg);
await this.bucketRef.fastRemove(path);
public async fastRemove(optionsArg: { path: string }) {
const path = plugins.path.join(this.getBasePath(), optionsArg.path);
await this.bucketRef.fastRemove({
path,
});
}
/**
* deletes the directory with all its contents
*/
public async deleteWithAllContents() {
public async delete() {
const deleteDirectory = async (directoryArg: Directory) => {
const childDirectories = await directoryArg.listDirectories();
if (childDirectories.length === 0) {
@ -218,7 +236,9 @@ export class Directory {
}
const files = await directoryArg.listFiles();
for (const file of files) {
await directoryArg.fastRemove(file.name);
await directoryArg.fastRemove({
path: file.name,
});
}
};
await deleteDirectory(this);

101
ts/smartbucket.classes.file.ts
View File

@ -7,29 +7,41 @@ export interface IFileMetaData {
size: string;
}
/**
* represents a file in a directory
*/
export class File {
// STATIC
public static async createFileFromString(
dirArg: Directory,
fileName: string,
fileContent: string
) {
await this.createFileFromBuffer(dirArg, fileName, Buffer.from(fileContent));
}
public static async createFileFromBuffer(
directoryRef: Directory,
fileName: string,
fileContent: Buffer
) {
const filePath = plugins.path.join(directoryRef.getBasePath(), fileName);
const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = directoryRef.bucketRef.smartbucketRef.minioClient
.putObject(this.name, filePath, streamIntake.getReadable())
.catch((e) => console.log(e));
streamIntake.pushData(fileContent);
streamIntake.signalEnd();
await putPromise;
/**
* creates a file in draft mode
* you need to call .save() to store it in s3
* @param optionsArg
*/
public static async create(optionsArg: {
directory: Directory;
name: string;
contents: Buffer | string | plugins.stream.Readable;
/**
* if contents are of type string, you can specify the encoding here
*/
encoding?: 'utf8' | 'binary';
}): Promise<File> {
const contents =
typeof optionsArg.contents === 'string'
? Buffer.from(optionsArg.contents, optionsArg.encoding)
: optionsArg.contents;
const file = new File({
directoryRefArg: optionsArg.directory,
fileName: optionsArg.name,
});
if (contents instanceof plugins.stream.Readable) {} else {
await optionsArg.directory.fastPut({
path: optionsArg.name,
contents: contents,
});
}
return file;
}
// INSTANCE
@ -39,9 +51,9 @@ export class File {
public path: string;
public metaData: IFileMetaData;
constructor(directoryRefArg: Directory, fileName: string) {
this.parentDirectoryRef = directoryRefArg;
this.name = fileName;
constructor(optionsArg: { directoryRefArg: Directory; fileName: string }) {
this.parentDirectoryRef = optionsArg.directoryRefArg;
this.name = optionsArg.fileName;
}
public async getContentAsString() {
@ -55,14 +67,16 @@ export class File {
.getObject(this.parentDirectoryRef.bucketRef.name, this.path)
.catch((e) => console.log(e));
let completeFile = Buffer.from('');
const duplexStream = plugins.smartstream.createDuplexStream<Buffer, Buffer>(
async (chunk) => {
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>(
{
writeFunction: async (chunk) => {
completeFile = Buffer.concat([chunk]);
return chunk;
},
async (cb) => {
finalFunction: async (cb) => {
done.resolve();
return Buffer.from('');
},
}
);
@ -75,7 +89,7 @@ export class File {
return completeFile;
}
public async streamContent() {
public async readStreaming() {
// TODO
throw new Error('not yet implemented');
}
@ -90,4 +104,37 @@ export class File {
);
await this.parentDirectoryRef.listFiles();
}
/**
* deletes the file
*/
public async delete() {}
/**
* allows locking the file
* @param optionsArg
*/
public async lock(optionsArg?: { timeoutMillis?: number }) {}
/**
* actively unlocks a file
*
*/
public async unlock(optionsArg?: {
/**
* unlock the file even if not locked from this instance
*/
force?: boolean;
}) {}
public async updateWithContents(optionsArg: {
contents: Buffer | string | plugins.stream.Readable;
encoding?: 'utf8' | 'binary';
}) {}
/**
* allows updating the metadata of a file
* @param updatedMetadata
*/
public async updateMetaData(updatedMetadata: any) {}
}