diff --git a/package.json b/package.json index beb1281..efcdda2 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,12 @@ "@push.rocks/tapbundle": "^5.0.23" }, "dependencies": { - "@push.rocks/smartmime": "^2.0.0", + "@push.rocks/smartmime": "^2.0.2", "@push.rocks/smartpath": "^5.0.18", "@push.rocks/smartpromise": "^4.0.3", "@push.rocks/smartrx": "^3.0.7", - "@push.rocks/smartstream": "^3.0.38", + "@push.rocks/smartstream": "^3.0.42", + "@push.rocks/smartunique": "^3.0.9", "@tsclass/tsclass": "^4.0.54", "minio": "^8.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2eee75e..20278b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@push.rocks/smartmime': - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^2.0.2 + version: 2.0.2 '@push.rocks/smartpath': specifier: ^5.0.18 version: 5.0.18 @@ -21,8 +21,11 @@ importers: specifier: ^3.0.7 version: 3.0.7 '@push.rocks/smartstream': - specifier: ^3.0.38 - version: 3.0.38 + specifier: ^3.0.42 + version: 3.0.42 + '@push.rocks/smartunique': + specifier: ^3.0.9 + version: 3.0.9 '@tsclass/tsclass': specifier: ^4.0.54 version: 4.0.54 @@ -35,10 +38,10 @@ importers: version: 2.1.80 '@git.zone/tsrun': specifier: ^1.2.46 - version: 1.2.46(@types/node@20.12.12) + version: 1.2.46(@types/node@20.14.0) '@git.zone/tstest': specifier: ^1.0.90 - version: 1.0.90(@types/node@20.12.12) + version: 1.0.90(@types/node@20.14.0) '@push.rocks/qenv': specifier: ^6.0.5 version: 6.0.5 @@ -412,8 +415,8 @@ packages: '@push.rocks/smartmime@1.0.6': resolution: {integrity: sha512-PHd+I4UcsnOATNg8wjDsSAmmJ4CwQFrQCNzd0HSJMs4ZpiK3Ya91almd6GLpDPU370U4HFh4FaPF4eEAI6vkJQ==} - '@push.rocks/smartmime@2.0.0': - resolution: {integrity: sha512-yNEYrQzWjxwinCT8djw9eFumpCIvIQQS9KXWLH0LT9COlFoaP/ruk7pogrGYKCo20tFITJyO6NmMCa24402rvA==} + '@push.rocks/smartmime@2.0.2': + resolution: {integrity: sha512-aXH1sFD73q9cEwPdeSeN7Zxd2aoVt9wE97ILFCW5gORylvm85Hgfq7SYkqykjQzEL8IDJKJF3G78+xcL2rALTg==} '@push.rocks/smartnetwork@3.0.2': resolution: {integrity: sha512-s6CNGzQ1n/d/6cOKXbxeW6/tO//dr1woLqI01g7XhqTriw0nsm2G2kWaZh2J0VOguGNWBgQVCIpR0LjdRNWb3g==} @@ -467,8 +470,8 @@ packages: '@push.rocks/smartstream@2.0.8': resolution: {integrity: sha512-GlF/9cCkvBHwKa3DK4DO5wjfSgqkj6gAS4TrY9uD5NMHu9RQv4WiNrElTYj7iCEpnZgUnLO3tzw1JA3NRIMnnA==} - '@push.rocks/smartstream@3.0.38': - resolution: {integrity: sha512-Sk9esPURWXldS0ZvgClCtrEyvELjvFnbQgUAelwoXWMfM8pXuB9BX1tE+Z1iBkB9Xyw2p1d9jYelO6waSXg0BQ==} + '@push.rocks/smartstream@3.0.42': + resolution: {integrity: sha512-5/laFoH8wCO7GAys8zRJGnYT2fvV4q83OzIxEZ6zcwTPLfP92WNmlTBezKrNYCVpc4gmmCVArdEX1Ha6Y3HnvA==} '@push.rocks/smartstring@4.0.15': resolution: {integrity: sha512-NTNeOjWyg+aHtBTiQEyXamr7oTvYZ3wS1fudHo9ua7CLrykpK+i+RxFyJaLg1zB5x9xQF3NLEQecB14HPFX8Cg==} @@ -765,6 +768,9 @@ packages: '@types/node@20.12.12': resolution: {integrity: sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==} + '@types/node@20.14.0': + resolution: {integrity: sha512-5cHBxFGJx6L4s56Bubp4fglrEpmyJypsqI6RgzMfBHWUJQGWAAi8cWcgetEbZXHYXo9C2Fa4EEds/uSyS4cxmA==} + '@types/parse5@6.0.3': resolution: {integrity: sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g==} @@ -3127,7 +3133,7 @@ snapshots: '@push.rocks/smartrequest': 2.0.22 '@push.rocks/smartrx': 3.0.7 '@push.rocks/smartsitemap': 2.0.3 - '@push.rocks/smartstream': 3.0.38 + '@push.rocks/smartstream': 3.0.42 '@push.rocks/smarttime': 4.0.6 '@push.rocks/taskbuffer': 3.1.7 '@push.rocks/webrequest': 3.0.37 @@ -3333,22 +3339,22 @@ snapshots: transitivePeerDependencies: - supports-color - '@git.zone/tsrun@1.2.46(@types/node@20.12.12)': + '@git.zone/tsrun@1.2.46(@types/node@20.14.0)': dependencies: '@push.rocks/smartfile': 10.0.41 '@push.rocks/smartshell': 3.0.5 - ts-node: 10.9.2(@types/node@20.12.12)(typescript@5.1.6) + ts-node: 10.9.2(@types/node@20.14.0)(typescript@5.1.6) typescript: 5.1.6 transitivePeerDependencies: - '@swc/core' - '@swc/wasm' - '@types/node' - '@git.zone/tstest@1.0.90(@types/node@20.12.12)': + '@git.zone/tstest@1.0.90(@types/node@20.14.0)': dependencies: '@api.global/typedserver': 3.0.37 '@git.zone/tsbundle': 2.0.15 - '@git.zone/tsrun': 1.2.46(@types/node@20.12.12) + '@git.zone/tsrun': 1.2.46(@types/node@20.14.0) '@push.rocks/consolecolor': 2.0.2 '@push.rocks/smartbrowser': 2.0.6 '@push.rocks/smartdelay': 3.0.5 @@ -3597,7 +3603,7 @@ snapshots: '@push.rocks/smartpath': 5.0.18 '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartrequest': 2.0.22 - '@push.rocks/smartstream': 3.0.38 + '@push.rocks/smartstream': 3.0.42 '@types/fs-extra': 11.0.4 '@types/glob': 8.1.0 '@types/js-yaml': 4.0.9 @@ -3664,7 +3670,7 @@ snapshots: '@types/mime-types': 2.1.4 mime-types: 2.1.35 - '@push.rocks/smartmime@2.0.0': + '@push.rocks/smartmime@2.0.2': dependencies: '@types/mime-types': 2.1.4 file-type: 19.0.0 @@ -3821,12 +3827,12 @@ snapshots: from2: 2.3.0 through2: 4.0.2 - '@push.rocks/smartstream@3.0.38': + '@push.rocks/smartstream@3.0.42': dependencies: '@push.rocks/lik': 6.0.15 + '@push.rocks/smartenv': 5.0.12 '@push.rocks/smartpromise': 4.0.3 '@push.rocks/smartrx': 3.0.7 - '@push.rocks/webstream': 1.0.8 '@push.rocks/smartstring@4.0.15': dependencies: @@ -4250,6 +4256,10 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@20.14.0': + dependencies: + undici-types: 5.26.5 + '@types/parse5@6.0.3': {} '@types/ping@0.4.4': {} @@ -6666,14 +6676,14 @@ snapshots: trough@2.2.0: {} - ts-node@10.9.2(@types/node@20.12.12)(typescript@5.1.6): + ts-node@10.9.2(@types/node@20.14.0)(typescript@5.1.6): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 20.12.12 + '@types/node': 20.14.0 acorn: 8.11.3 acorn-walk: 8.3.2 arg: 4.1.3 diff --git a/test/test.ts b/test/test.ts index d4fbbfa..41f1970 100644 --- a/test/test.ts +++ b/test/test.ts @@ -45,7 +45,7 @@ tap.test('should get data in bucket', async () => { }); const fileStringStream = await myBucket.fastGetStream({ path: 'hithere/socool.txt', - }); + }, 'nodestream'); console.log(fileString); }); diff --git a/ts/00_commitinfo_data.ts b/ts/00_commitinfo_data.ts index cc09131..b43b1a3 100644 --- a/ts/00_commitinfo_data.ts +++ b/ts/00_commitinfo_data.ts @@ -3,6 +3,6 @@ */ export const commitinfo = { name: '@push.rocks/smartbucket', - version: '3.0.9', + version: '3.0.10', description: 'A TypeScript library offering simple and cloud-agnostic object storage with advanced features like bucket creation, file and directory management, and data streaming.' } diff --git a/ts/classes.bucket.ts b/ts/classes.bucket.ts index e0dc6b7..3681dd8 100644 --- a/ts/classes.bucket.ts +++ b/ts/classes.bucket.ts @@ -107,10 +107,12 @@ export class Bucket { /** * get file */ - public async fastGet(optionsArg: Parameters[0]): Promise { + public async fastGet(optionsArg: { + path: string + }): Promise { const done = plugins.smartpromise.defer(); let completeFile: Buffer; - const replaySubject = await this.fastGetStream(optionsArg); + const replaySubject = await this.fastGetReplaySubject(optionsArg); const subscription = replaySubject.subscribe({ next: (chunk) => { if (completeFile) { @@ -131,7 +133,13 @@ export class Bucket { return completeFile; } - public async fastGetStream(optionsArg: { + /** + * good when time to first byte is important + * and multiple subscribers are expected + * @param optionsArg + * @returns + */ + public async fastGetReplaySubject(optionsArg: { path: string; }): Promise> { const fileStream = await this.smartbucketRef.minioClient @@ -161,12 +169,54 @@ export class Bucket { return replaySubject; } + public fastGetStream(optionsArg: { + path: string; + }, typeArg: 'webstream'): Promise + public async fastGetStream(optionsArg: { + path: string; + }, typeArg: 'nodestream'): Promise + + /** + * fastGetStream + * @param optionsArg + * @returns + */ + public async fastGetStream(optionsArg: { path: string; }, typeArg: 'webstream' | 'nodestream' = 'nodestream'): Promise{ + const fileStream = await this.smartbucketRef.minioClient + .getObject(this.name, optionsArg.path) + .catch((e) => console.log(e)); + const duplexStream = new plugins.smartstream.SmartDuplex({ + writeFunction: async (chunk) => { + return chunk; + }, + finalFunction: async (cb) => { + return null; + } + }); + + if (!fileStream) { + return null; + } + + const smartstream = new plugins.smartstream.StreamWrapper([ + fileStream, + duplexStream, + ]); + smartstream.run(); + if (typeArg === 'nodestream') { + return duplexStream; + }; + if (typeArg === 'webstream') { + return (await duplexStream.getWebStreams()).readable; + } + } + /** * store file as stream */ public async fastPutStream(optionsArg: { path: string; - dataStream: plugins.stream.Readable; + dataStream: plugins.stream.Readable | ReadableStream; nativeMetadata?: { [key: string]: string }; overwrite?: boolean; }): Promise { @@ -182,12 +232,14 @@ export class Bucket { } else { console.log(`Creating new object at path '${optionsArg.path}' in bucket '${this.name}'.`); } + + const streamIntake = await plugins.smartstream.StreamIntake.fromStream(optionsArg.dataStream); // Proceed with putting the object await this.smartbucketRef.minioClient.putObject( this.name, optionsArg.path, - optionsArg.dataStream, + streamIntake, null, ...(optionsArg.nativeMetadata ? (() => { @@ -313,6 +365,13 @@ export class Bucket { } } + /** + * deletes this bucket + */ + public async delete() { + await this.smartbucketRef.minioClient.removeBucket(this.name); + } + public async fastStat(pathDescriptor: interfaces.IPathDecriptor) { let checkPath = await helpers.reducePathDescriptorToPath(pathDescriptor); return this.smartbucketRef.minioClient.statObject(this.name, checkPath); diff --git a/ts/classes.directory.ts b/ts/classes.directory.ts index caa4d1f..876c2b0 100644 --- a/ts/classes.directory.ts +++ b/ts/classes.directory.ts @@ -234,14 +234,30 @@ export class Directory { return result; } - public async fastGetStream(pathArg: string): Promise> { - const path = plugins.path.join(this.getBasePath(), pathArg); + public fastGetStream(optionsArg: { + path: string; + }, typeArg: 'webstream'): Promise + public async fastGetStream(optionsArg: { + path: string; + }, typeArg: 'nodestream'): Promise + + /** + * fastGetStream + * @param optionsArg + * @returns + */ + public async fastGetStream(optionsArg: { path: string; }, typeArg: 'webstream' | 'nodestream'): Promise{ + const path = plugins.path.join(this.getBasePath(), optionsArg.path); const result = await this.bucketRef.fastGetStream({ - path, - }); + path + }, typeArg as any); return result; } + /** + * removes a file within the directory + * @param optionsArg + */ public async fastRemove(optionsArg: { path: string }) { const path = plugins.path.join(this.getBasePath(), optionsArg.path); await this.bucketRef.fastRemove({ diff --git a/ts/classes.file.ts b/ts/classes.file.ts index 1d61cbd..f4c2192 100644 --- a/ts/classes.file.ts +++ b/ts/classes.file.ts @@ -4,7 +4,6 @@ import * as interfaces from './interfaces.js'; import { Directory } from './classes.directory.js'; import { MetaData } from './classes.metadata.js'; - /** * represents a file in a directory */ @@ -33,7 +32,8 @@ export class File { directoryRefArg: optionsArg.directory, fileName: optionsArg.name, }); - if (contents instanceof plugins.stream.Readable) {} else { + if (contents instanceof plugins.stream.Readable) { + } else { await optionsArg.directory.fastPut({ path: optionsArg.name, contents: contents, @@ -48,7 +48,7 @@ export class File { public getBasePath(): string { return plugins.path.join(this.parentDirectoryRef.getBasePath(), this.name); - }; + } constructor(optionsArg: { directoryRefArg: Directory; fileName: string }) { this.parentDirectoryRef = optionsArg.directoryRefArg; @@ -67,35 +67,61 @@ export class File { return resultBuffer; } - public async getReadStream() { - const readStream = this.parentDirectoryRef.bucketRef.fastGetStream({ - path: this.getBasePath(), - }); + public async getReadStream(typeArg: 'webstream'): Promise; + public async getReadStream(typeArg: 'nodestream'): Promise; + public async getReadStream( + typeArg: 'nodestream' | 'webstream' + ): Promise { + const readStream = this.parentDirectoryRef.bucketRef.fastGetStream( + { + path: this.getBasePath(), + }, + typeArg as any + ); + return readStream; } /** - * removes this file - * for using recycling mechanics use .delete() + * deletes this file */ - public async remove() { - await this.parentDirectoryRef.bucketRef.fastRemove({ - path: this.getBasePath(), - }); - if (!this.name.endsWith('.metadata')) { + public async delete(optionsArg?: { + mode: 'trash' | 'permanent'; + }) { + + optionsArg = { + ... { + mode: 'permanent', + }, + ...optionsArg, + } + + if (optionsArg.mode === 'permanent') { await this.parentDirectoryRef.bucketRef.fastRemove({ - path: this.getBasePath() + '.metadata', + path: this.getBasePath(), + }); + if (!this.name.endsWith('.metadata')) { + const metadata = await this.getMetaData(); + await metadata.metadataFile.delete(optionsArg); + } + } else if (optionsArg.mode === 'trash') { + const metadata = await this.getMetaData(); + await metadata.storeCustomMetaData({ + key: 'recycle', + value: { + deletedAt: Date.now(), + originalPath: this.getBasePath(), + }, + }); + const trashName = plugins.smartunique.uuid4(); + await this.move({ + directory: await this.parentDirectoryRef.bucketRef.getBaseDirectory(), + path: plugins.path.join('trash', trashName), }); } + await this.parentDirectoryRef.listFiles(); } - /** - * deletes the file with recycling mechanics - */ - public async delete() { - await this.remove(); - } - /** * allows locking the file * @param optionsArg @@ -125,10 +151,13 @@ export class File { } public async updateWithContents(optionsArg: { - contents: Buffer | string | plugins.stream.Readable; + contents: Buffer | string | plugins.stream.Readable | ReadableStream; encoding?: 'utf8' | 'binary'; }) { - if (optionsArg.contents instanceof plugins.stream.Readable) { + if ( + optionsArg.contents instanceof plugins.stream.Readable || + optionsArg.contents instanceof ReadableStream + ) { await this.parentDirectoryRef.bucketRef.fastPutStream({ path: this.getBasePath(), dataStream: optionsArg.contents, diff --git a/ts/plugins.ts b/ts/plugins.ts index cf4c08c..b8f67a0 100644 --- a/ts/plugins.ts +++ b/ts/plugins.ts @@ -10,8 +10,9 @@ import * as smartpath from '@push.rocks/smartpath'; import * as smartpromise from '@push.rocks/smartpromise'; import * as smartrx from '@push.rocks/smartrx'; import * as smartstream from '@push.rocks/smartstream'; +import * as smartunique from '@push.rocks/smartunique'; -export { smartmime, smartpath, smartpromise, smartrx, smartstream }; +export { smartmime, smartpath, smartpromise, smartrx, smartstream, smartunique }; // @tsclass import * as tsclass from '@tsclass/tsclass';