Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fac59e24d7 | |||
| 18bdb5c7c2 |
@@ -1,5 +1,13 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-03-14 - 4.5.0 - feat(storage)
|
||||||
|
generalize S3 client and watcher interfaces to storage-oriented naming with backward compatibility
|
||||||
|
|
||||||
|
- rename SmartBucket internals from s3Client to storageClient and normalize configuration using storage descriptor types
|
||||||
|
- update watcher and interface types from S3-specific names to storage-oriented equivalents while keeping deprecated aliases
|
||||||
|
- refresh tests and documentation comments to reflect S3-compatible object storage terminology
|
||||||
|
- bump build, test, AWS SDK, and related dependency versions
|
||||||
|
|
||||||
## 2026-01-25 - 4.4.1 - fix(tests)
|
## 2026-01-25 - 4.4.1 - fix(tests)
|
||||||
add explicit 'as string' type assertions to environment variable retrievals in tests
|
add explicit 'as string' type assertions to environment variable retrievals in tests
|
||||||
|
|
||||||
|
|||||||
17
package.json
17
package.json
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@push.rocks/smartbucket",
|
"name": "@push.rocks/smartbucket",
|
||||||
"version": "4.4.1",
|
"version": "4.5.0",
|
||||||
"description": "A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.",
|
"description": "A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.",
|
||||||
"main": "dist_ts/index.js",
|
"main": "dist_ts/index.js",
|
||||||
"typings": "dist_ts/index.d.ts",
|
"typings": "dist_ts/index.d.ts",
|
||||||
@@ -12,23 +12,24 @@
|
|||||||
"build": "(tsbuild tsfolders --allowimplicitany)"
|
"build": "(tsbuild tsfolders --allowimplicitany)"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@git.zone/tsbuild": "^4.1.2",
|
"@git.zone/tsbuild": "^4.3.0",
|
||||||
"@git.zone/tsrun": "^2.0.1",
|
"@git.zone/tsrun": "^2.0.1",
|
||||||
"@git.zone/tstest": "^3.1.6",
|
"@git.zone/tstest": "^3.3.2",
|
||||||
"@push.rocks/qenv": "^6.1.3",
|
"@push.rocks/qenv": "^6.1.3",
|
||||||
"@push.rocks/tapbundle": "^6.0.3"
|
"@push.rocks/tapbundle": "^6.0.3",
|
||||||
|
"@types/node": "^22.15.29"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@aws-sdk/client-s3": "^3.975.0",
|
"@aws-sdk/client-s3": "^3.1009.0",
|
||||||
"@push.rocks/smartmime": "^2.0.4",
|
"@push.rocks/smartmime": "^2.0.4",
|
||||||
"@push.rocks/smartpath": "^6.0.0",
|
"@push.rocks/smartpath": "^6.0.0",
|
||||||
"@push.rocks/smartpromise": "^4.2.3",
|
"@push.rocks/smartpromise": "^4.2.3",
|
||||||
"@push.rocks/smartrx": "^3.0.10",
|
"@push.rocks/smartrx": "^3.0.10",
|
||||||
"@push.rocks/smartstream": "^3.2.5",
|
"@push.rocks/smartstream": "^3.4.0",
|
||||||
"@push.rocks/smartstring": "^4.1.0",
|
"@push.rocks/smartstring": "^4.1.0",
|
||||||
"@push.rocks/smartunique": "^3.0.9",
|
"@push.rocks/smartunique": "^3.0.9",
|
||||||
"@tsclass/tsclass": "^9.3.0",
|
"@tsclass/tsclass": "^9.4.0",
|
||||||
"minimatch": "^10.1.1"
|
"minimatch": "^10.2.4"
|
||||||
},
|
},
|
||||||
"private": false,
|
"private": false,
|
||||||
"files": [
|
"files": [
|
||||||
|
|||||||
3557
pnpm-lock.yaml
generated
3557
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,7 @@ import { expect, tap } from '@git.zone/tstest/tapbundle';
|
|||||||
import * as plugins from '../ts/plugins.js';
|
import * as plugins from '../ts/plugins.js';
|
||||||
import * as smartbucket from '../ts/index.js';
|
import * as smartbucket from '../ts/index.js';
|
||||||
|
|
||||||
class FakeS3Client {
|
class FakeStorageClient {
|
||||||
private callIndex = 0;
|
private callIndex = 0;
|
||||||
|
|
||||||
constructor(private readonly pages: Array<Partial<plugins.s3.ListObjectsV2Output>>) {}
|
constructor(private readonly pages: Array<Partial<plugins.s3.ListObjectsV2Output>>) {}
|
||||||
@@ -30,7 +30,7 @@ tap.test('MetaData.hasMetaData should return false when metadata file does not e
|
|||||||
});
|
});
|
||||||
|
|
||||||
tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => {
|
tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => {
|
||||||
const fakeSmartbucket = { s3Client: new FakeS3Client([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket;
|
const fakeSmartbucket = { storageClient: new FakeStorageClient([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket;
|
||||||
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
||||||
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
||||||
|
|
||||||
@@ -51,7 +51,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
|
|||||||
Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })),
|
Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })),
|
||||||
IsTruncated: false,
|
IsTruncated: false,
|
||||||
};
|
};
|
||||||
const fakeSmartbucket = { s3Client: new FakeS3Client([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket;
|
const fakeSmartbucket = { storageClient: new FakeStorageClient([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket;
|
||||||
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
||||||
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
||||||
|
|
||||||
@@ -61,7 +61,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
|
|||||||
|
|
||||||
tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => {
|
tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => {
|
||||||
const fakeSmartbucket = {
|
const fakeSmartbucket = {
|
||||||
s3Client: new FakeS3Client([
|
storageClient: new FakeStorageClient([
|
||||||
{ CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' },
|
{ CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' },
|
||||||
{ CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false },
|
{ CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false },
|
||||||
]),
|
]),
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||||
import * as smartbucket from '../ts/index.js';
|
import * as smartbucket from '../ts/index.js';
|
||||||
import type { IS3ChangeEvent } from '../ts/interfaces.js';
|
import type { IStorageChangeEvent } from '../ts/interfaces.js';
|
||||||
|
|
||||||
// Get test configuration
|
// Get test configuration
|
||||||
import * as qenv from '@push.rocks/qenv';
|
import * as qenv from '@push.rocks/qenv';
|
||||||
@@ -57,7 +57,7 @@ tap.test('should create watcher with custom options', async () => {
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should detect add events for new files', async () => {
|
tap.test('should detect add events for new files', async () => {
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-test/',
|
prefix: 'watcher-test/',
|
||||||
pollIntervalMs: 500,
|
pollIntervalMs: 500,
|
||||||
@@ -94,7 +94,7 @@ tap.test('should detect add events for new files', async () => {
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should detect modify events for changed files', async () => {
|
tap.test('should detect modify events for changed files', async () => {
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-test/',
|
prefix: 'watcher-test/',
|
||||||
pollIntervalMs: 500,
|
pollIntervalMs: 500,
|
||||||
@@ -131,7 +131,7 @@ tap.test('should detect modify events for changed files', async () => {
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should detect delete events for removed files', async () => {
|
tap.test('should detect delete events for removed files', async () => {
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-test/',
|
prefix: 'watcher-test/',
|
||||||
pollIntervalMs: 500,
|
pollIntervalMs: 500,
|
||||||
@@ -174,7 +174,7 @@ tap.test('should emit initial state as add events when includeInitial is true',
|
|||||||
contents: 'content 2',
|
contents: 'content 2',
|
||||||
});
|
});
|
||||||
|
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-initial/',
|
prefix: 'watcher-initial/',
|
||||||
pollIntervalMs: 10000, // Long interval - we only care about initial events
|
pollIntervalMs: 10000, // Long interval - we only care about initial events
|
||||||
@@ -206,13 +206,13 @@ tap.test('should emit initial state as add events when includeInitial is true',
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should emit events via EventEmitter pattern', async () => {
|
tap.test('should emit events via EventEmitter pattern', async () => {
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-emitter/',
|
prefix: 'watcher-emitter/',
|
||||||
pollIntervalMs: 500,
|
pollIntervalMs: 500,
|
||||||
});
|
});
|
||||||
|
|
||||||
watcher.on('change', (event: IS3ChangeEvent) => {
|
watcher.on('change', (event: IStorageChangeEvent) => {
|
||||||
events.push(event);
|
events.push(event);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -239,7 +239,7 @@ tap.test('should emit events via EventEmitter pattern', async () => {
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should buffer events when bufferTimeMs is set', async () => {
|
tap.test('should buffer events when bufferTimeMs is set', async () => {
|
||||||
const bufferedEvents: (IS3ChangeEvent | IS3ChangeEvent[])[] = [];
|
const bufferedEvents: (IStorageChangeEvent | IStorageChangeEvent[])[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-buffer/',
|
prefix: 'watcher-buffer/',
|
||||||
pollIntervalMs: 200,
|
pollIntervalMs: 200,
|
||||||
@@ -327,8 +327,8 @@ tap.test('should stop gracefully with stop()', async () => {
|
|||||||
await watcher.stop();
|
await watcher.stop();
|
||||||
|
|
||||||
// Watcher should not poll after stop
|
// Watcher should not poll after stop
|
||||||
const eventsCaptured: IS3ChangeEvent[] = [];
|
const eventsCaptured: IStorageChangeEvent[] = [];
|
||||||
watcher.on('change', (event: IS3ChangeEvent) => {
|
watcher.on('change', (event: IStorageChangeEvent) => {
|
||||||
eventsCaptured.push(event);
|
eventsCaptured.push(event);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -362,7 +362,7 @@ tap.test('should stop gracefully with close() alias', async () => {
|
|||||||
// ==========================
|
// ==========================
|
||||||
|
|
||||||
tap.test('should only detect changes within specified prefix', async () => {
|
tap.test('should only detect changes within specified prefix', async () => {
|
||||||
const events: IS3ChangeEvent[] = [];
|
const events: IStorageChangeEvent[] = [];
|
||||||
const watcher = testBucket.createWatcher({
|
const watcher = testBucket.createWatcher({
|
||||||
prefix: 'watcher-prefix-a/',
|
prefix: 'watcher-prefix-a/',
|
||||||
pollIntervalMs: 500,
|
pollIntervalMs: 500,
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
*/
|
*/
|
||||||
export const commitinfo = {
|
export const commitinfo = {
|
||||||
name: '@push.rocks/smartbucket',
|
name: '@push.rocks/smartbucket',
|
||||||
version: '4.4.1',
|
version: '4.5.0',
|
||||||
description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.'
|
description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.'
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,12 +13,12 @@ import { BucketWatcher } from './classes.watcher.js';
|
|||||||
/**
|
/**
|
||||||
* The bucket class exposes the basic functionality of a bucket.
|
* The bucket class exposes the basic functionality of a bucket.
|
||||||
* The functions of the bucket alone are enough to
|
* The functions of the bucket alone are enough to
|
||||||
* operate in S3 basic fashion on blobs of data.
|
* operate on blobs of data in an S3-compatible object store.
|
||||||
*/
|
*/
|
||||||
export class Bucket {
|
export class Bucket {
|
||||||
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string): Promise<Bucket> {
|
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string): Promise<Bucket> {
|
||||||
const command = new plugins.s3.ListBucketsCommand({});
|
const command = new plugins.s3.ListBucketsCommand({});
|
||||||
const buckets = await smartbucketRef.s3Client.send(command);
|
const buckets = await smartbucketRef.storageClient.send(command);
|
||||||
const foundBucket = buckets.Buckets!.find((bucket) => bucket.Name === bucketNameArg);
|
const foundBucket = buckets.Buckets!.find((bucket) => bucket.Name === bucketNameArg);
|
||||||
|
|
||||||
if (foundBucket) {
|
if (foundBucket) {
|
||||||
@@ -32,13 +32,13 @@ export class Bucket {
|
|||||||
|
|
||||||
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||||
const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName });
|
const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName });
|
||||||
await smartbucketRef.s3Client.send(command);
|
await smartbucketRef.storageClient.send(command);
|
||||||
return new Bucket(smartbucketRef, bucketName);
|
return new Bucket(smartbucketRef, bucketName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||||
const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName });
|
const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName });
|
||||||
await smartbucketRef.s3Client.send(command);
|
await smartbucketRef.storageClient.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
public smartbucketRef: SmartBucket;
|
public smartbucketRef: SmartBucket;
|
||||||
@@ -112,7 +112,7 @@ export class Bucket {
|
|||||||
Key: reducedPath,
|
Key: reducedPath,
|
||||||
Body: optionsArg.contents,
|
Body: optionsArg.contents,
|
||||||
});
|
});
|
||||||
await this.smartbucketRef.s3Client.send(command);
|
await this.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
||||||
const parsedPath = plugins.path.parse(reducedPath);
|
const parsedPath = plugins.path.parse(reducedPath);
|
||||||
@@ -172,7 +172,7 @@ export class Bucket {
|
|||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: optionsArg.path,
|
Key: optionsArg.path,
|
||||||
});
|
});
|
||||||
const response = await this.smartbucketRef.s3Client.send(command);
|
const response = await this.smartbucketRef.storageClient.send(command);
|
||||||
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
||||||
|
|
||||||
// Convert the stream to a format that supports piping
|
// Convert the stream to a format that supports piping
|
||||||
@@ -216,7 +216,7 @@ export class Bucket {
|
|||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: optionsArg.path,
|
Key: optionsArg.path,
|
||||||
});
|
});
|
||||||
const response = await this.smartbucketRef.s3Client.send(command);
|
const response = await this.smartbucketRef.storageClient.send(command);
|
||||||
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||||
|
|
||||||
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
||||||
@@ -272,7 +272,7 @@ export class Bucket {
|
|||||||
Body: optionsArg.readableStream,
|
Body: optionsArg.readableStream,
|
||||||
Metadata: optionsArg.nativeMetadata,
|
Metadata: optionsArg.nativeMetadata,
|
||||||
});
|
});
|
||||||
await this.smartbucketRef.s3Client.send(command);
|
await this.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
||||||
@@ -297,7 +297,7 @@ export class Bucket {
|
|||||||
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
|
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
|
||||||
|
|
||||||
// Retrieve current object information to use in copy conditions
|
// Retrieve current object information to use in copy conditions
|
||||||
const currentObjInfo = await this.smartbucketRef.s3Client.send(
|
const currentObjInfo = await this.smartbucketRef.storageClient.send(
|
||||||
new plugins.s3.HeadObjectCommand({
|
new plugins.s3.HeadObjectCommand({
|
||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: optionsArg.sourcePath,
|
Key: optionsArg.sourcePath,
|
||||||
@@ -319,7 +319,7 @@ export class Bucket {
|
|||||||
Metadata: newNativeMetadata,
|
Metadata: newNativeMetadata,
|
||||||
MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
|
MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
|
||||||
});
|
});
|
||||||
await this.smartbucketRef.s3Client.send(command);
|
await this.smartbucketRef.storageClient.send(command);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Error updating metadata:', err);
|
console.error('Error updating metadata:', err);
|
||||||
throw err; // rethrow to allow caller to handle
|
throw err; // rethrow to allow caller to handle
|
||||||
@@ -379,7 +379,7 @@ export class Bucket {
|
|||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: optionsArg.path,
|
Key: optionsArg.path,
|
||||||
});
|
});
|
||||||
await this.smartbucketRef.s3Client.send(command);
|
await this.smartbucketRef.storageClient.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -393,7 +393,7 @@ export class Bucket {
|
|||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: optionsArg.path,
|
Key: optionsArg.path,
|
||||||
});
|
});
|
||||||
await this.smartbucketRef.s3Client.send(command);
|
await this.smartbucketRef.storageClient.send(command);
|
||||||
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
||||||
return true;
|
return true;
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
@@ -411,7 +411,7 @@ export class Bucket {
|
|||||||
* deletes this bucket
|
* deletes this bucket
|
||||||
*/
|
*/
|
||||||
public async delete() {
|
public async delete() {
|
||||||
await this.smartbucketRef.s3Client.send(
|
await this.smartbucketRef.storageClient.send(
|
||||||
new plugins.s3.DeleteBucketCommand({ Bucket: this.name })
|
new plugins.s3.DeleteBucketCommand({ Bucket: this.name })
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -422,7 +422,7 @@ export class Bucket {
|
|||||||
Bucket: this.name,
|
Bucket: this.name,
|
||||||
Key: checkPath,
|
Key: checkPath,
|
||||||
});
|
});
|
||||||
return this.smartbucketRef.s3Client.send(command);
|
return this.smartbucketRef.storageClient.send(command);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
||||||
@@ -432,7 +432,7 @@ export class Bucket {
|
|||||||
Prefix: checkPath,
|
Prefix: checkPath,
|
||||||
Delimiter: '/',
|
Delimiter: '/',
|
||||||
});
|
});
|
||||||
const { CommonPrefixes } = await this.smartbucketRef.s3Client.send(command);
|
const { CommonPrefixes } = await this.smartbucketRef.storageClient.send(command);
|
||||||
return !!CommonPrefixes && CommonPrefixes.length > 0;
|
return !!CommonPrefixes && CommonPrefixes.length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,7 +443,7 @@ export class Bucket {
|
|||||||
Prefix: checkPath,
|
Prefix: checkPath,
|
||||||
Delimiter: '/',
|
Delimiter: '/',
|
||||||
});
|
});
|
||||||
const { Contents } = await this.smartbucketRef.s3Client.send(command);
|
const { Contents } = await this.smartbucketRef.storageClient.send(command);
|
||||||
return !!Contents && Contents.length > 0;
|
return !!Contents && Contents.length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -454,7 +454,7 @@ export class Bucket {
|
|||||||
Key: optionsArg.path,
|
Key: optionsArg.path,
|
||||||
Range: `bytes=0-${optionsArg.length - 1}`,
|
Range: `bytes=0-${optionsArg.length - 1}`,
|
||||||
});
|
});
|
||||||
const response = await this.smartbucketRef.s3Client.send(command);
|
const response = await this.smartbucketRef.storageClient.send(command);
|
||||||
const chunks: Buffer[] = [];
|
const chunks: Buffer[] = [];
|
||||||
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||||
|
|
||||||
@@ -497,7 +497,7 @@ export class Bucket {
|
|||||||
ContinuationToken: continuationToken,
|
ContinuationToken: continuationToken,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.smartbucketRef.s3Client.send(command);
|
const response = await this.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
for (const obj of response.Contents || []) {
|
for (const obj of response.Contents || []) {
|
||||||
if (obj.Key) yield obj.Key;
|
if (obj.Key) yield obj.Key;
|
||||||
@@ -531,7 +531,7 @@ export class Bucket {
|
|||||||
ContinuationToken: token,
|
ContinuationToken: token,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.smartbucketRef.s3Client.send(command);
|
const response = await this.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
for (const obj of response.Contents || []) {
|
for (const obj of response.Contents || []) {
|
||||||
if (obj.Key) subscriber.next(obj.Key);
|
if (obj.Key) subscriber.next(obj.Key);
|
||||||
@@ -646,7 +646,7 @@ export class Bucket {
|
|||||||
|
|
||||||
// Explicitly type the response
|
// Explicitly type the response
|
||||||
const response: plugins.s3.ListObjectsV2Output =
|
const response: plugins.s3.ListObjectsV2Output =
|
||||||
await this.smartbucketRef.s3Client.send(listCommand);
|
await this.smartbucketRef.storageClient.send(listCommand);
|
||||||
|
|
||||||
console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
|
console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
|
||||||
|
|
||||||
@@ -660,7 +660,7 @@ export class Bucket {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await this.smartbucketRef.s3Client.send(deleteCommand);
|
await this.smartbucketRef.storageClient.send(deleteCommand);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update continuation token and truncation status
|
// Update continuation token and truncation status
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ export class Directory {
|
|||||||
Delimiter: delimiter,
|
Delimiter: delimiter,
|
||||||
ContinuationToken: continuationToken,
|
ContinuationToken: continuationToken,
|
||||||
});
|
});
|
||||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
if (response.Contents) {
|
if (response.Contents) {
|
||||||
allContents.push(...response.Contents);
|
allContents.push(...response.Contents);
|
||||||
@@ -213,7 +213,7 @@ export class Directory {
|
|||||||
Prefix: this.getBasePath(),
|
Prefix: this.getBasePath(),
|
||||||
Delimiter: '/',
|
Delimiter: '/',
|
||||||
});
|
});
|
||||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||||
return response.Contents;
|
return response.Contents;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,12 +222,12 @@ export class Directory {
|
|||||||
*/
|
*/
|
||||||
public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
|
public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
|
||||||
/**
|
/**
|
||||||
* in s3 a directory does not exist if it is empty
|
* in object storage a directory does not exist if it is empty
|
||||||
* this option returns a directory even if it is empty
|
* this option returns a directory even if it is empty
|
||||||
*/
|
*/
|
||||||
getEmptyDirectory?: boolean;
|
getEmptyDirectory?: boolean;
|
||||||
/**
|
/**
|
||||||
* in s3 a directory does not exist if it is empty
|
* in object storage a directory does not exist if it is empty
|
||||||
* this option creates a directory even if it is empty using a initializer file
|
* this option creates a directory even if it is empty using a initializer file
|
||||||
*/
|
*/
|
||||||
createWithInitializerFile?: boolean;
|
createWithInitializerFile?: boolean;
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ export class File {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* creates a file in draft mode
|
* creates a file in draft mode
|
||||||
* you need to call .save() to store it in s3
|
* you need to call .save() to store it in object storage
|
||||||
* @param optionsArg
|
* @param optionsArg
|
||||||
*/
|
*/
|
||||||
public static async create(optionsArg: {
|
public static async create(optionsArg: {
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ export class ListCursor {
|
|||||||
ContinuationToken: this.continuationToken,
|
ContinuationToken: this.continuationToken,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.bucket.smartbucketRef.s3Client.send(command);
|
const response = await this.bucket.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
const keys = (response.Contents || [])
|
const keys = (response.Contents || [])
|
||||||
.map((obj) => obj.Key)
|
.map((obj) => obj.Key)
|
||||||
|
|||||||
@@ -2,26 +2,28 @@
|
|||||||
|
|
||||||
import * as plugins from './plugins.js';
|
import * as plugins from './plugins.js';
|
||||||
import { Bucket } from './classes.bucket.js';
|
import { Bucket } from './classes.bucket.js';
|
||||||
import { normalizeS3Descriptor } from './helpers.js';
|
import { normalizeStorageDescriptor } from './helpers.js';
|
||||||
|
|
||||||
export class SmartBucket {
|
export class SmartBucket {
|
||||||
public config: plugins.tsclass.storage.IS3Descriptor;
|
public config: plugins.tsclass.storage.IStorageDescriptor;
|
||||||
|
|
||||||
public s3Client: plugins.s3.S3Client;
|
public storageClient: plugins.s3.S3Client;
|
||||||
|
|
||||||
|
/** @deprecated Use storageClient instead */
|
||||||
|
public get s3Client(): plugins.s3.S3Client {
|
||||||
|
return this.storageClient;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* the constructor of SmartBucket
|
* the constructor of SmartBucket
|
||||||
*/
|
*/
|
||||||
/**
|
constructor(configArg: plugins.tsclass.storage.IStorageDescriptor) {
|
||||||
* the constructor of SmartBucket
|
|
||||||
*/
|
|
||||||
constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
|
|
||||||
this.config = configArg;
|
this.config = configArg;
|
||||||
|
|
||||||
// Use the normalizer to handle various endpoint formats
|
// Use the normalizer to handle various endpoint formats
|
||||||
const { normalized } = normalizeS3Descriptor(configArg);
|
const { normalized } = normalizeStorageDescriptor(configArg);
|
||||||
|
|
||||||
this.s3Client = new plugins.s3.S3Client({
|
this.storageClient = new plugins.s3.S3Client({
|
||||||
endpoint: normalized.endpointUrl,
|
endpoint: normalized.endpointUrl,
|
||||||
region: normalized.region,
|
region: normalized.region,
|
||||||
credentials: normalized.credentials,
|
credentials: normalized.credentials,
|
||||||
@@ -47,7 +49,7 @@ export class SmartBucket {
|
|||||||
*/
|
*/
|
||||||
public async bucketExists(bucketNameArg: string): Promise<boolean> {
|
public async bucketExists(bucketNameArg: string): Promise<boolean> {
|
||||||
const command = new plugins.s3.ListBucketsCommand({});
|
const command = new plugins.s3.ListBucketsCommand({});
|
||||||
const buckets = await this.s3Client.send(command);
|
const buckets = await this.storageClient.send(command);
|
||||||
return buckets.Buckets?.some(bucket => bucket.Name === bucketNameArg) ?? false;
|
return buckets.Buckets?.some(bucket => bucket.Name === bucketNameArg) ?? false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import type { Bucket } from './classes.bucket.js';
|
|||||||
import { EventEmitter } from 'node:events';
|
import { EventEmitter } from 'node:events';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* BucketWatcher monitors an S3 bucket for changes (add/modify/delete)
|
* BucketWatcher monitors a storage bucket for changes (add/modify/delete)
|
||||||
* using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern.
|
* using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern.
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
@@ -34,11 +34,11 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
public readyDeferred = plugins.smartpromise.defer();
|
public readyDeferred = plugins.smartpromise.defer();
|
||||||
|
|
||||||
/** Observable for receiving change events (supports RxJS operators) */
|
/** Observable for receiving change events (supports RxJS operators) */
|
||||||
public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IS3ChangeEvent | interfaces.IS3ChangeEvent[]>;
|
public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IStorageChangeEvent | interfaces.IStorageChangeEvent[]>;
|
||||||
|
|
||||||
// Internal subjects and state
|
// Internal subjects and state
|
||||||
private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>;
|
private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>;
|
||||||
private previousState: Map<string, interfaces.IS3ObjectState>;
|
private previousState: Map<string, interfaces.IStorageObjectState>;
|
||||||
private pollIntervalId: ReturnType<typeof setInterval> | null = null;
|
private pollIntervalId: ReturnType<typeof setInterval> | null = null;
|
||||||
private isPolling = false;
|
private isPolling = false;
|
||||||
private isStopped = false;
|
private isStopped = false;
|
||||||
@@ -65,13 +65,13 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
this.previousState = new Map();
|
this.previousState = new Map();
|
||||||
|
|
||||||
// Initialize raw subject for emitting changes
|
// Initialize raw subject for emitting changes
|
||||||
this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>();
|
this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>();
|
||||||
|
|
||||||
// Configure the public observable with optional buffering
|
// Configure the public observable with optional buffering
|
||||||
if (this.bufferTimeMs && this.bufferTimeMs > 0) {
|
if (this.bufferTimeMs && this.bufferTimeMs > 0) {
|
||||||
this.changeSubject = this.rawSubject.pipe(
|
this.changeSubject = this.rawSubject.pipe(
|
||||||
plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs),
|
plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs),
|
||||||
plugins.smartrx.rxjs.ops.filter((events: interfaces.IS3ChangeEvent[]) => events.length > 0)
|
plugins.smartrx.rxjs.ops.filter((events: interfaces.IStorageChangeEvent[]) => events.length > 0)
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
this.changeSubject = this.rawSubject.asObservable();
|
this.changeSubject = this.rawSubject.asObservable();
|
||||||
@@ -174,7 +174,7 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
// Build current state
|
// Build current state
|
||||||
const currentState = new Map<string, interfaces.IS3ObjectState>();
|
const currentState = new Map<string, interfaces.IStorageObjectState>();
|
||||||
|
|
||||||
for await (const obj of this.listObjectsWithMetadata()) {
|
for await (const obj of this.listObjectsWithMetadata()) {
|
||||||
if (this.isStopped) {
|
if (this.isStopped) {
|
||||||
@@ -205,7 +205,7 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
/**
|
/**
|
||||||
* Detect changes between current and previous state
|
* Detect changes between current and previous state
|
||||||
*/
|
*/
|
||||||
private detectChanges(currentState: Map<string, interfaces.IS3ObjectState>): void {
|
private detectChanges(currentState: Map<string, interfaces.IStorageObjectState>): void {
|
||||||
// Detect added and modified objects
|
// Detect added and modified objects
|
||||||
for (const [key, current] of currentState) {
|
for (const [key, current] of currentState) {
|
||||||
const previous = this.previousState.get(key);
|
const previous = this.previousState.get(key);
|
||||||
@@ -253,7 +253,7 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
/**
|
/**
|
||||||
* Emit a change event via both RxJS Subject and EventEmitter
|
* Emit a change event via both RxJS Subject and EventEmitter
|
||||||
*/
|
*/
|
||||||
private emitChange(event: interfaces.IS3ChangeEvent): void {
|
private emitChange(event: interfaces.IStorageChangeEvent): void {
|
||||||
this.rawSubject.next(event);
|
this.rawSubject.next(event);
|
||||||
this.emit('change', event);
|
this.emit('change', event);
|
||||||
}
|
}
|
||||||
@@ -277,7 +277,7 @@ export class BucketWatcher extends EventEmitter {
|
|||||||
ContinuationToken: continuationToken,
|
ContinuationToken: continuationToken,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||||
|
|
||||||
for (const obj of response.Contents || []) {
|
for (const obj of response.Contents || []) {
|
||||||
yield obj;
|
yield obj;
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.I
|
|||||||
return returnPath;
|
return returnPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
// S3 Descriptor Normalization
|
// Storage Descriptor Normalization
|
||||||
export interface IS3Warning {
|
export interface IStorageWarning {
|
||||||
code: string;
|
code: string;
|
||||||
message: string;
|
message: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface INormalizedS3Config {
|
export interface INormalizedStorageConfig {
|
||||||
endpointUrl: string;
|
endpointUrl: string;
|
||||||
host: string;
|
host: string;
|
||||||
protocol: 'http' | 'https';
|
protocol: 'http' | 'https';
|
||||||
@@ -40,7 +40,7 @@ export interface INormalizedS3Config {
|
|||||||
forcePathStyle: boolean;
|
forcePathStyle: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IS3Warning } {
|
function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IStorageWarning } {
|
||||||
if (typeof value === 'boolean') return { value };
|
if (typeof value === 'boolean') return { value };
|
||||||
if (typeof value === 'string') {
|
if (typeof value === 'string') {
|
||||||
const v = value.trim().toLowerCase();
|
const v = value.trim().toLowerCase();
|
||||||
@@ -66,7 +66,7 @@ function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warni
|
|||||||
return { value: undefined };
|
return { value: undefined };
|
||||||
}
|
}
|
||||||
|
|
||||||
function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IS3Warning } {
|
function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IStorageWarning } {
|
||||||
if (port === undefined || port === null || port === '') return { value: undefined };
|
if (port === undefined || port === null || port === '') return { value: undefined };
|
||||||
const n = typeof port === 'number' ? port : Number(String(port).trim());
|
const n = typeof port === 'number' ? port : Number(String(port).trim());
|
||||||
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) {
|
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) {
|
||||||
@@ -81,8 +81,8 @@ function coercePortMaybe(port: unknown): { value: number | undefined; warning?:
|
|||||||
return { value: n };
|
return { value: n };
|
||||||
}
|
}
|
||||||
|
|
||||||
function sanitizeEndpointString(raw: unknown): { value: string; warnings: IS3Warning[] } {
|
function sanitizeEndpointString(raw: unknown): { value: string; warnings: IStorageWarning[] } {
|
||||||
const warnings: IS3Warning[] = [];
|
const warnings: IStorageWarning[] = [];
|
||||||
let s = String(raw ?? '').trim();
|
let s = String(raw ?? '').trim();
|
||||||
if (s !== String(raw ?? '')) {
|
if (s !== String(raw ?? '')) {
|
||||||
warnings.push({
|
warnings.push({
|
||||||
@@ -138,17 +138,17 @@ function parseEndpointHostPort(
|
|||||||
return { hadScheme, host, port, extras };
|
return { hadScheme, host, port, extras };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function normalizeS3Descriptor(
|
export function normalizeStorageDescriptor(
|
||||||
input: plugins.tsclass.storage.IS3Descriptor,
|
input: plugins.tsclass.storage.IStorageDescriptor,
|
||||||
logger?: { warn: (msg: string) => void }
|
logger?: { warn: (msg: string) => void }
|
||||||
): { normalized: INormalizedS3Config; warnings: IS3Warning[] } {
|
): { normalized: INormalizedStorageConfig; warnings: IStorageWarning[] } {
|
||||||
const warnings: IS3Warning[] = [];
|
const warnings: IStorageWarning[] = [];
|
||||||
const logWarn = (w: IS3Warning) => {
|
const logWarn = (w: IStorageWarning) => {
|
||||||
warnings.push(w);
|
warnings.push(w);
|
||||||
if (logger) {
|
if (logger) {
|
||||||
logger.warn(`[SmartBucket S3] ${w.code}: ${w.message}`);
|
logger.warn(`[SmartBucket] ${w.code}: ${w.message}`);
|
||||||
} else {
|
} else {
|
||||||
console.warn(`[SmartBucket S3] ${w.code}: ${w.message}`);
|
console.warn(`[SmartBucket] ${w.code}: ${w.message}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -163,7 +163,7 @@ export function normalizeS3Descriptor(
|
|||||||
endpointSanWarnings.forEach(logWarn);
|
endpointSanWarnings.forEach(logWarn);
|
||||||
|
|
||||||
if (!endpointStr) {
|
if (!endpointStr) {
|
||||||
throw new Error('S3 endpoint is required (got empty string). Provide hostname or URL.');
|
throw new Error('Storage endpoint is required (got empty string). Provide hostname or URL.');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Provisional protocol selection for parsing host:port forms
|
// Provisional protocol selection for parsing host:port forms
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ export interface IPathDecriptor {
|
|||||||
// ================================
|
// ================================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Internal state tracking for an S3 object
|
* Internal state tracking for a storage object
|
||||||
*/
|
*/
|
||||||
export interface IS3ObjectState {
|
export interface IStorageObjectState {
|
||||||
key: string;
|
key: string;
|
||||||
etag: string;
|
etag: string;
|
||||||
size: number;
|
size: number;
|
||||||
@@ -22,7 +22,7 @@ export interface IS3ObjectState {
|
|||||||
/**
|
/**
|
||||||
* Change event emitted by BucketWatcher
|
* Change event emitted by BucketWatcher
|
||||||
*/
|
*/
|
||||||
export interface IS3ChangeEvent {
|
export interface IStorageChangeEvent {
|
||||||
type: 'add' | 'modify' | 'delete';
|
type: 'add' | 'modify' | 'delete';
|
||||||
key: string;
|
key: string;
|
||||||
size?: number;
|
size?: number;
|
||||||
@@ -54,4 +54,14 @@ export interface IBucketWatcherOptions {
|
|||||||
pageSize?: number;
|
pageSize?: number;
|
||||||
// Future websocket options will be added here
|
// Future websocket options will be added here
|
||||||
// websocketUrl?: string;
|
// websocketUrl?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ================================
|
||||||
|
// Deprecated aliases
|
||||||
|
// ================================
|
||||||
|
|
||||||
|
/** @deprecated Use IStorageObjectState instead */
|
||||||
|
export type IS3ObjectState = IStorageObjectState;
|
||||||
|
|
||||||
|
/** @deprecated Use IStorageChangeEvent instead */
|
||||||
|
export type IS3ChangeEvent = IStorageChangeEvent;
|
||||||
|
|||||||
Reference in New Issue
Block a user