push.rocks/smartbucket
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BREAKING CHANGE(core): update

Browse Source
This commit is contained in:
Philipp Kunz 2024-05-17 18:53:11 +02:00
parent dab74572b8
commit a0be96bf23
8 changed files with 13609 additions and 14048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17907
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

17
package.json
View File

@ -12,20 +12,19 @@
"build": "(tsbuild --web --allowimplicitany)" "build": "(tsbuild --web --allowimplicitany)"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbuild": "^2.1.63", "@git.zone/tsbuild": "^2.1.76",
"@git.zone/tsrun": "^1.2.46", "@git.zone/tsrun": "^1.2.46",
"@git.zone/tstest": "^1.0.71", "@git.zone/tstest": "^1.0.90",
"@push.rocks/qenv": "^6.0.4", "@push.rocks/qenv": "^6.0.5",
"@push.rocks/tapbundle": "^5.0.3" "@push.rocks/tapbundle": "^5.0.23"
}, },
"dependencies": { "dependencies": {
"@push.rocks/smartpath": "^5.0.5", "@push.rocks/smartpath": "^5.0.18",
"@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartpromise": "^4.0.3",
"@push.rocks/smartrx": "^3.0.7", "@push.rocks/smartrx": "^3.0.7",
"@push.rocks/smartstream": "^2.0.2", "@push.rocks/smartstream": "^3.0.37",
"@tsclass/tsclass": "^4.0.50", "@tsclass/tsclass": "^4.0.54",
"@types/minio": "^7.0.13", "minio": "^8.0.0"
"minio": "^7.0.28"
}, },
"private": false, "private": false,
"files": [ "files": [

9362
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

65
test/test.ts
View File

@ -33,28 +33,55 @@ tap.test('should get a bucket', async () => {
// Fast operations // Fast operations
tap.test('should store data in bucket fast', async () => { tap.test('should store data in bucket fast', async () => {
await myBucket.fastPut('hithere/socool.txt', 'hi there!'); await myBucket.fastPut({
path: 'hithere/socool.txt',
contents: 'hi there!',
});
}); });
tap.test('should get data in bucket', async () => { tap.test('should get data in bucket', async () => {
const fileString = await myBucket.fastGet('hithere/socool.txt'); const fileString = await myBucket.fastGet({
const fileStringStream = await myBucket.fastGetStream('hithere/socool.txt'); path: 'hithere/socool.txt',
});
const fileStringStream = await myBucket.fastGetStream({
path: 'hithere/socool.txt',
});
console.log(fileString); console.log(fileString);
}); });
tap.test('should delete data in bucket', async () => { tap.test('should delete data in bucket', async () => {
await myBucket.fastRemove('hithere/socool.txt'); await myBucket.fastRemove({
path: 'hithere/socool.txt',
});
}); });
// fs operations // fs operations
tap.test('prepare for directory style tests', async () => { tap.test('prepare for directory style tests', async () => {
await myBucket.fastPut('dir1/file1.txt', 'dir1/file1.txt content'); await myBucket.fastPut({
await myBucket.fastPut('dir1/file2.txt', 'dir1/file2.txt content'); path: 'dir1/file1.txt',
await myBucket.fastPut('dir2/file1.txt', 'dir2/file1.txt content'); contents: 'dir1/file1.txt content',
await myBucket.fastPut('dir3/file1.txt', 'dir3/file1.txt content'); });
await myBucket.fastPut('dir3/dir4/file1.txt', 'dir3/dir4/file1.txt content'); await myBucket.fastPut({
await myBucket.fastPut('file1.txt', 'file1 content'); path: 'dir1/file2.txt',
contents: 'dir1/file2.txt content',
});
await myBucket.fastPut({
path: 'dir2/file1.txt',
contents: 'dir2/file1.txt content',
});
await myBucket.fastPut({
path: 'dir3/file1.txt',
contents: 'dir3/file1.txt content',
});
await myBucket.fastPut({
path: 'dir3/dir4/file1.txt',
contents: 'dir3/dir4/file1.txt content',
});
await myBucket.fastPut({
path: 'file1.txt',
contents: 'file1 content',
});
}); });
tap.test('should get base directory', async () => { tap.test('should get base directory', async () => {
@ -77,12 +104,18 @@ tap.test('should correctly build paths for sub directories', async () => {
}); });
tap.test('clean up directory style tests', async () => { tap.test('clean up directory style tests', async () => {
await myBucket.fastRemove('dir1/file1.txt'); await myBucket.fastRemove({
await myBucket.fastRemove('dir1/file2.txt'); path: 'dir1/file1.txt',
await myBucket.fastRemove('dir2/file1.txt'); });
await myBucket.fastRemove('dir3/file1.txt'); await myBucket.fastRemove({
await myBucket.fastRemove('dir3/dir4/file1.txt'); path: 'dir1/file2.txt',
await myBucket.fastRemove('file1.txt'); });
await myBucket.fastRemove({
path: 'dir2/file1.txt',
});
await myBucket.fastRemove({ path: 'dir3/file1.txt' });
await myBucket.fastRemove({ path: 'dir3/dir4/file1.txt' });
await myBucket.fastRemove({ path: 'file1.txt' });
}); });
tap.start(); tap.start();

2
ts/00_commitinfo_data.ts
View File

@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartbucket', name: '@push.rocks/smartbucket',
version: '2.0.5', version: '3.0.0',
description: 'A TypeScript library for simple cloud independent object storage with support for buckets, directories, and files.' description: 'A TypeScript library for simple cloud independent object storage with support for buckets, directories, and files.'
} }

123
ts/smartbucket.classes.bucket.ts
View File

@ -49,58 +49,63 @@ export class Bucket {
/** /**
* store file * store file
*/ */
public async fastPut(pathArg: string, fileContent: string | Buffer): Promise<void> { public async fastPut(optionsArg: {
path: string;
contents: string | Buffer;
}): Promise<void> {
const streamIntake = new plugins.smartstream.StreamIntake(); const streamIntake = new plugins.smartstream.StreamIntake();
const putPromise = this.smartbucketRef.minioClient const putPromise = this.smartbucketRef.minioClient
.putObject(this.name, pathArg, streamIntake.getReadable()) .putObject(this.name, optionsArg.path, streamIntake)
.catch((e) => console.log(e)); .catch((e) => console.log(e));
streamIntake.pushData(fileContent); streamIntake.pushData(optionsArg.contents);
streamIntake.signalEnd(); streamIntake.signalEnd();
await putPromise; const response = await putPromise;
} }
/** /**
* get file * get file
*/ */
public async fastGet(pathArg: string): Promise<Buffer> { public async fastGet(optionsArg: Parameters<typeof this.fastGetStream>[0]): Promise<Buffer> {
const done = plugins.smartpromise.defer(); const done = plugins.smartpromise.defer();
let completeFile: Buffer; let completeFile: Buffer;
const replaySubject = await this.fastGetStream(pathArg); const replaySubject = await this.fastGetStream(optionsArg);
const subscription = replaySubject.subscribe( const subscription = replaySubject.subscribe({
(chunk) => { next: (chunk) => {
if (completeFile) { if (completeFile) {
completeFile = Buffer.concat([completeFile, chunk]); completeFile = Buffer.concat([completeFile, chunk]);
} else { } else {
completeFile = chunk; completeFile = chunk;
} }
}, },
(err) => { complete: () => {
console.log(err);
},
() => {
done.resolve(); done.resolve();
subscription.unsubscribe(); subscription.unsubscribe();
} },
); error: (err) => {
console.log(err);
},
});
await done.promise; await done.promise;
return completeFile; return completeFile;
} }
public async fastGetStream(pathArg: string): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> { public async fastGetStream(optionsArg: {
path: string;
}): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const fileStream = await this.smartbucketRef.minioClient const fileStream = await this.smartbucketRef.minioClient
.getObject(this.name, pathArg) .getObject(this.name, optionsArg.path)
.catch((e) => console.log(e)); .catch((e) => console.log(e));
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>(); const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
const duplexStream = plugins.smartstream.createDuplexStream<Buffer, Buffer>( const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
async (chunk) => { writeFunction: async (chunk) => {
replaySubject.next(chunk); replaySubject.next(chunk);
return chunk; return chunk;
}, },
async (cb) => { finalFunction: async (cb) => {
replaySubject.complete(); replaySubject.complete();
return Buffer.from(''); return Buffer.from('');
} }
); });
if (!fileStream) { if (!fileStream) {
return null; return null;
@ -109,7 +114,6 @@ export class Bucket {
const smartstream = new plugins.smartstream.StreamWrapper([ const smartstream = new plugins.smartstream.StreamWrapper([
fileStream, fileStream,
duplexStream, duplexStream,
plugins.smartstream.cleanPipe(),
]); ]);
smartstream.run(); smartstream.run();
return replaySubject; return replaySubject;
@ -119,16 +123,16 @@ export class Bucket {
* store file as stream * store file as stream
*/ */
public async fastPutStream(optionsArg: { public async fastPutStream(optionsArg: {
pathArg: string; path: string;
dataStream: plugins.stream.Readable; dataStream: plugins.stream.Readable;
metadata?: { [key: string]: string }; nativeMetadata?: { [key: string]: string };
}): Promise<void> { }): Promise<void> {
await this.smartbucketRef.minioClient.putObject( await this.smartbucketRef.minioClient.putObject(
this.name, this.name,
optionsArg.pathArg, optionsArg.path,
optionsArg.dataStream, optionsArg.dataStream,
null, null,
...(optionsArg.metadata ...(optionsArg.nativeMetadata
? (() => { ? (() => {
const returnObject: any = {}; const returnObject: any = {};
return returnObject; return returnObject;
@ -137,30 +141,47 @@ export class Bucket {
); );
} }
public async updateMetadata( public async copyObject(optionsArg: {
bucket: string, /**
objectKey: string, * the
metadata: { [key: string]: string } */
): Promise<void> { objectKey: string;
/**
* in case you want to copy to another bucket specify it here
*/
targetBucket?: Bucket;
targetBucketKey?: string;
/**
* metadata will be merged with existing metadata
*/
nativeMetadata?: { [key: string]: string };
deleteExistingNativeMetadata?: boolean;
}): Promise<void> {
try { try {
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
// Retrieve current object information to use in copy conditions // Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.minioClient.statObject(bucket, objectKey); const currentObjInfo = await this.smartbucketRef.minioClient.statObject(
targetBucketName,
optionsArg.objectKey
);
// Setting up copy conditions // Setting up copy conditions
const copyConditions = new plugins.minio.CopyConditions(); const copyConditions = new plugins.minio.CopyConditions();
// Prepare new metadata, merging current and new metadata // Prepare new metadata
const newMetadata = { const newNativeMetadata = {
...currentObjInfo.metaData, ...(optionsArg.deleteExistingNativeMetadata ? {} : currentObjInfo.metaData),
...metadata, ...optionsArg.nativeMetadata,
}; };
// Define the copy operation as a Promise // Define the copy operation as a Promise
// TODO: check on issue here: https://github.com/minio/minio-js/issues/1286
await this.smartbucketRef.minioClient.copyObject( await this.smartbucketRef.minioClient.copyObject(
bucket, this.name,
objectKey, optionsArg.objectKey,
`/${bucket}/${objectKey}`, `/${targetBucketName}/${optionsArg.objectKey}`,
copyConditions, copyConditions
); );
} catch (err) { } catch (err) {
console.error('Error updating metadata:', err); console.error('Error updating metadata:', err);
@ -171,7 +192,27 @@ export class Bucket {
/** /**
* removeObject * removeObject
*/ */
public async fastRemove(pathArg: string) { public async fastRemove(optionsArg: {
await this.smartbucketRef.minioClient.removeObject(this.name, pathArg); path: string;
}) {
await this.smartbucketRef.minioClient.removeObject(this.name, optionsArg.path);
}
public async doesObjectExist(optionsArg: {
path: string;
}): Promise<boolean> {
try {
await this.smartbucketRef.minioClient.statObject(this.name, optionsArg.path);
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
return true;
} catch (error) {
if (error.code === 'NotFound') {
console.log(`Object '${optionsArg.path}' does not exist in bucket '${this.name}'.`);
return false;
} else {
console.error('Error checking object existence:', error);
throw error; // Rethrow if it's not a NotFound error to handle unexpected issues
}
}
} }
} }

64
ts/smartbucket.classes.directory.ts
View File

@ -70,8 +70,8 @@ export class Directory {
false false
); );
const fileArray: File[] = []; const fileArray: File[] = [];
const duplexStream = plugins.smartstream.createDuplexStream<plugins.minio.BucketItem, void>( const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
async (bucketItem) => { writeFunction: async (bucketItem) => {
if (bucketItem.prefix) { if (bucketItem.prefix) {
return; return;
} }
@ -83,13 +83,18 @@ export class Directory {
subtractedPath = subtractedPath.substr(1); subtractedPath = subtractedPath.substr(1);
} }
if (!subtractedPath.includes('/')) { if (!subtractedPath.includes('/')) {
fileArray.push(new File(this, subtractedPath)); fileArray.push(
new File({
directoryRefArg: this,
fileName: subtractedPath,
})
);
} }
}, },
async (tools) => { finalFunction: async (tools) => {
done.resolve(); done.resolve();
} }
); });
fileNameStream.pipe(duplexStream); fileNameStream.pipe(duplexStream);
await done.promise; await done.promise;
return fileArray; return fileArray;
@ -107,8 +112,8 @@ export class Directory {
false false
); );
const directoryArray: Directory[] = []; const directoryArray: Directory[] = [];
const duplexStream = plugins.smartstream.createDuplexStream<plugins.minio.BucketItem, void>( const duplexStream = new plugins.smartstream.SmartDuplex<plugins.minio.BucketItem, void>({
async (bucketItem) => { writeFunction: async (bucketItem) => {
if (bucketItem.name) { if (bucketItem.name) {
return; return;
} }
@ -124,10 +129,10 @@ export class Directory {
directoryArray.push(new Directory(this.bucketRef, this, dirName)); directoryArray.push(new Directory(this.bucketRef, this, dirName));
} }
}, },
async (tools) => { finalFunction: async (tools) => {
done.resolve(); done.resolve();
} }
); });
completeDirStream.pipe(duplexStream); completeDirStream.pipe(duplexStream);
await done.promise; await done.promise;
return directoryArray; return directoryArray;
@ -177,36 +182,49 @@ export class Directory {
* @param relativePathArg * @param relativePathArg
*/ */
public async createEmptyFile(relativePathArg: string) { public async createEmptyFile(relativePathArg: string) {
const emtpyFile = await File.createFileFromString(this, relativePathArg, ''); const emtpyFile = await File.create({
directory: this,
name: relativePathArg,
contents: '',
});
} }
// file operations // file operations
public async fastStore(pathArg: string, contentArg: string | Buffer) { public async fastPut(optionsArg: { path: string; contents: string | Buffer }) {
const path = plugins.path.join(this.getBasePath(), pathArg); const path = plugins.path.join(this.getBasePath(), optionsArg.path);
await this.bucketRef.fastPut(path, contentArg); await this.bucketRef.fastPut({
path,
contents: optionsArg.contents,
});
} }
public async fastGet(pathArg: string) { public async fastGet(optionsArg: { path: string }) {
const path = plugins.path.join(this.getBasePath(), pathArg); const path = plugins.path.join(this.getBasePath(), optionsArg.path);
const result = await this.bucketRef.fastGet(path); const result = await this.bucketRef.fastGet({
path,
});
return result; return result;
} }
public async fastGetStream(pathArg: string): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> { public async fastGetStream(pathArg: string): Promise<plugins.smartrx.rxjs.ReplaySubject<Buffer>> {
const path = plugins.path.join(this.getBasePath(), pathArg); const path = plugins.path.join(this.getBasePath(), pathArg);
const result = await this.bucketRef.fastGetStream(path); const result = await this.bucketRef.fastGetStream({
path,
});
return result; return result;
} }
public async fastRemove(pathArg: string) { public async fastRemove(optionsArg: { path: string }) {
const path = plugins.path.join(this.getBasePath(), pathArg); const path = plugins.path.join(this.getBasePath(), optionsArg.path);
await this.bucketRef.fastRemove(path); await this.bucketRef.fastRemove({
path,
});
} }
/** /**
* deletes the directory with all its contents * deletes the directory with all its contents
*/ */
public async deleteWithAllContents() { public async delete() {
const deleteDirectory = async (directoryArg: Directory) => { const deleteDirectory = async (directoryArg: Directory) => {
const childDirectories = await directoryArg.listDirectories(); const childDirectories = await directoryArg.listDirectories();
if (childDirectories.length === 0) { if (childDirectories.length === 0) {
@ -218,7 +236,9 @@ export class Directory {
} }
const files = await directoryArg.listFiles(); const files = await directoryArg.listFiles();
for (const file of files) { for (const file of files) {
await directoryArg.fastRemove(file.name); await directoryArg.fastRemove({
path: file.name,
});
} }
}; };
await deleteDirectory(this); await deleteDirectory(this);

111
ts/smartbucket.classes.file.ts
View File

@ -7,29 +7,41 @@ export interface IFileMetaData {
size: string; size: string;
} }
/**
* represents a file in a directory
*/
export class File { export class File {
// STATIC // STATIC
public static async createFileFromString(
dirArg: Directory,
fileName: string,
fileContent: string
) {
await this.createFileFromBuffer(dirArg, fileName, Buffer.from(fileContent));
}
public static async createFileFromBuffer( /**
directoryRef: Directory, * creates a file in draft mode
fileName: string, * you need to call .save() to store it in s3
fileContent: Buffer * @param optionsArg
) { */
const filePath = plugins.path.join(directoryRef.getBasePath(), fileName); public static async create(optionsArg: {
const streamIntake = new plugins.smartstream.StreamIntake(); directory: Directory;
const putPromise = directoryRef.bucketRef.smartbucketRef.minioClient name: string;
.putObject(this.name, filePath, streamIntake.getReadable()) contents: Buffer | string | plugins.stream.Readable;
.catch((e) => console.log(e)); /**
streamIntake.pushData(fileContent); * if contents are of type string, you can specify the encoding here
streamIntake.signalEnd(); */
await putPromise; encoding?: 'utf8' | 'binary';
}): Promise<File> {
const contents =
typeof optionsArg.contents === 'string'
? Buffer.from(optionsArg.contents, optionsArg.encoding)
: optionsArg.contents;
const file = new File({
directoryRefArg: optionsArg.directory,
fileName: optionsArg.name,
});
if (contents instanceof plugins.stream.Readable) {} else {
await optionsArg.directory.fastPut({
path: optionsArg.name,
contents: contents,
});
}
return file;
} }
// INSTANCE // INSTANCE
@ -39,9 +51,9 @@ export class File {
public path: string; public path: string;
public metaData: IFileMetaData; public metaData: IFileMetaData;
constructor(directoryRefArg: Directory, fileName: string) { constructor(optionsArg: { directoryRefArg: Directory; fileName: string }) {
this.parentDirectoryRef = directoryRefArg; this.parentDirectoryRef = optionsArg.directoryRefArg;
this.name = fileName; this.name = optionsArg.fileName;
} }
public async getContentAsString() { public async getContentAsString() {
@ -55,14 +67,16 @@ export class File {
.getObject(this.parentDirectoryRef.bucketRef.name, this.path) .getObject(this.parentDirectoryRef.bucketRef.name, this.path)
.catch((e) => console.log(e)); .catch((e) => console.log(e));
let completeFile = Buffer.from(''); let completeFile = Buffer.from('');
const duplexStream = plugins.smartstream.createDuplexStream<Buffer, Buffer>( const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>(
async (chunk) => { {
completeFile = Buffer.concat([chunk]); writeFunction: async (chunk) => {
return chunk; completeFile = Buffer.concat([chunk]);
}, return chunk;
async (cb) => { },
done.resolve(); finalFunction: async (cb) => {
return Buffer.from(''); done.resolve();
return Buffer.from('');
},
} }
); );
@ -75,7 +89,7 @@ export class File {
return completeFile; return completeFile;
} }
public async streamContent() { public async readStreaming() {
// TODO // TODO
throw new Error('not yet implemented'); throw new Error('not yet implemented');
} }
@ -90,4 +104,37 @@ export class File {
); );
await this.parentDirectoryRef.listFiles(); await this.parentDirectoryRef.listFiles();
} }
/**
* deletes the file
*/
public async delete() {}
/**
* allows locking the file
* @param optionsArg
*/
public async lock(optionsArg?: { timeoutMillis?: number }) {}
/**
* actively unlocks a file
*
*/
public async unlock(optionsArg?: {
/**
* unlock the file even if not locked from this instance
*/
force?: boolean;
}) {}
public async updateWithContents(optionsArg: {
contents: Buffer | string | plugins.stream.Readable;
encoding?: 'utf8' | 'binary';
}) {}
/**
* allows updating the metadata of a file
* @param updatedMetadata
*/
public async updateMetaData(updatedMetadata: any) {}
} }