feat(storage): generalize S3 client and watcher interfaces to storage-oriented naming with backward compatibility
This commit is contained in:
@@ -1,5 +1,13 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-03-14 - 4.5.0 - feat(storage)
|
||||
generalize S3 client and watcher interfaces to storage-oriented naming with backward compatibility
|
||||
|
||||
- rename SmartBucket internals from s3Client to storageClient and normalize configuration using storage descriptor types
|
||||
- update watcher and interface types from S3-specific names to storage-oriented equivalents while keeping deprecated aliases
|
||||
- refresh tests and documentation comments to reflect S3-compatible object storage terminology
|
||||
- bump build, test, AWS SDK, and related dependency versions
|
||||
|
||||
## 2026-01-25 - 4.4.1 - fix(tests)
|
||||
add explicit 'as string' type assertions to environment variable retrievals in tests
|
||||
|
||||
|
||||
15
package.json
15
package.json
@@ -12,23 +12,24 @@
|
||||
"build": "(tsbuild tsfolders --allowimplicitany)"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@git.zone/tsbuild": "^4.1.2",
|
||||
"@git.zone/tsbuild": "^4.3.0",
|
||||
"@git.zone/tsrun": "^2.0.1",
|
||||
"@git.zone/tstest": "^3.1.6",
|
||||
"@git.zone/tstest": "^3.3.2",
|
||||
"@push.rocks/qenv": "^6.1.3",
|
||||
"@push.rocks/tapbundle": "^6.0.3"
|
||||
"@push.rocks/tapbundle": "^6.0.3",
|
||||
"@types/node": "^22.15.29"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.975.0",
|
||||
"@aws-sdk/client-s3": "^3.1009.0",
|
||||
"@push.rocks/smartmime": "^2.0.4",
|
||||
"@push.rocks/smartpath": "^6.0.0",
|
||||
"@push.rocks/smartpromise": "^4.2.3",
|
||||
"@push.rocks/smartrx": "^3.0.10",
|
||||
"@push.rocks/smartstream": "^3.2.5",
|
||||
"@push.rocks/smartstream": "^3.4.0",
|
||||
"@push.rocks/smartstring": "^4.1.0",
|
||||
"@push.rocks/smartunique": "^3.0.9",
|
||||
"@tsclass/tsclass": "^9.3.0",
|
||||
"minimatch": "^10.1.1"
|
||||
"@tsclass/tsclass": "^9.4.0",
|
||||
"minimatch": "^10.2.4"
|
||||
},
|
||||
"private": false,
|
||||
"files": [
|
||||
|
||||
3557
pnpm-lock.yaml
generated
3557
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,7 @@ import { expect, tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from '../ts/plugins.js';
|
||||
import * as smartbucket from '../ts/index.js';
|
||||
|
||||
class FakeS3Client {
|
||||
class FakeStorageClient {
|
||||
private callIndex = 0;
|
||||
|
||||
constructor(private readonly pages: Array<Partial<plugins.s3.ListObjectsV2Output>>) {}
|
||||
@@ -30,7 +30,7 @@ tap.test('MetaData.hasMetaData should return false when metadata file does not e
|
||||
});
|
||||
|
||||
tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => {
|
||||
const fakeSmartbucket = { s3Client: new FakeS3Client([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket;
|
||||
const fakeSmartbucket = { storageClient: new FakeStorageClient([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket;
|
||||
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
||||
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
||||
|
||||
@@ -51,7 +51,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
|
||||
Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })),
|
||||
IsTruncated: false,
|
||||
};
|
||||
const fakeSmartbucket = { s3Client: new FakeS3Client([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket;
|
||||
const fakeSmartbucket = { storageClient: new FakeStorageClient([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket;
|
||||
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
|
||||
const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
|
||||
|
||||
@@ -61,7 +61,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
|
||||
|
||||
tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => {
|
||||
const fakeSmartbucket = {
|
||||
s3Client: new FakeS3Client([
|
||||
storageClient: new FakeStorageClient([
|
||||
{ CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' },
|
||||
{ CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false },
|
||||
]),
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as smartbucket from '../ts/index.js';
|
||||
import type { IS3ChangeEvent } from '../ts/interfaces.js';
|
||||
import type { IStorageChangeEvent } from '../ts/interfaces.js';
|
||||
|
||||
// Get test configuration
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
@@ -57,7 +57,7 @@ tap.test('should create watcher with custom options', async () => {
|
||||
// ==========================
|
||||
|
||||
tap.test('should detect add events for new files', async () => {
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-test/',
|
||||
pollIntervalMs: 500,
|
||||
@@ -94,7 +94,7 @@ tap.test('should detect add events for new files', async () => {
|
||||
// ==========================
|
||||
|
||||
tap.test('should detect modify events for changed files', async () => {
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-test/',
|
||||
pollIntervalMs: 500,
|
||||
@@ -131,7 +131,7 @@ tap.test('should detect modify events for changed files', async () => {
|
||||
// ==========================
|
||||
|
||||
tap.test('should detect delete events for removed files', async () => {
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-test/',
|
||||
pollIntervalMs: 500,
|
||||
@@ -174,7 +174,7 @@ tap.test('should emit initial state as add events when includeInitial is true',
|
||||
contents: 'content 2',
|
||||
});
|
||||
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-initial/',
|
||||
pollIntervalMs: 10000, // Long interval - we only care about initial events
|
||||
@@ -206,13 +206,13 @@ tap.test('should emit initial state as add events when includeInitial is true',
|
||||
// ==========================
|
||||
|
||||
tap.test('should emit events via EventEmitter pattern', async () => {
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-emitter/',
|
||||
pollIntervalMs: 500,
|
||||
});
|
||||
|
||||
watcher.on('change', (event: IS3ChangeEvent) => {
|
||||
watcher.on('change', (event: IStorageChangeEvent) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
@@ -239,7 +239,7 @@ tap.test('should emit events via EventEmitter pattern', async () => {
|
||||
// ==========================
|
||||
|
||||
tap.test('should buffer events when bufferTimeMs is set', async () => {
|
||||
const bufferedEvents: (IS3ChangeEvent | IS3ChangeEvent[])[] = [];
|
||||
const bufferedEvents: (IStorageChangeEvent | IStorageChangeEvent[])[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-buffer/',
|
||||
pollIntervalMs: 200,
|
||||
@@ -327,8 +327,8 @@ tap.test('should stop gracefully with stop()', async () => {
|
||||
await watcher.stop();
|
||||
|
||||
// Watcher should not poll after stop
|
||||
const eventsCaptured: IS3ChangeEvent[] = [];
|
||||
watcher.on('change', (event: IS3ChangeEvent) => {
|
||||
const eventsCaptured: IStorageChangeEvent[] = [];
|
||||
watcher.on('change', (event: IStorageChangeEvent) => {
|
||||
eventsCaptured.push(event);
|
||||
});
|
||||
|
||||
@@ -362,7 +362,7 @@ tap.test('should stop gracefully with close() alias', async () => {
|
||||
// ==========================
|
||||
|
||||
tap.test('should only detect changes within specified prefix', async () => {
|
||||
const events: IS3ChangeEvent[] = [];
|
||||
const events: IStorageChangeEvent[] = [];
|
||||
const watcher = testBucket.createWatcher({
|
||||
prefix: 'watcher-prefix-a/',
|
||||
pollIntervalMs: 500,
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
*/
|
||||
export const commitinfo = {
|
||||
name: '@push.rocks/smartbucket',
|
||||
version: '4.4.1',
|
||||
version: '4.5.0',
|
||||
description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.'
|
||||
}
|
||||
|
||||
@@ -13,12 +13,12 @@ import { BucketWatcher } from './classes.watcher.js';
|
||||
/**
|
||||
* The bucket class exposes the basic functionality of a bucket.
|
||||
* The functions of the bucket alone are enough to
|
||||
* operate in S3 basic fashion on blobs of data.
|
||||
* operate on blobs of data in an S3-compatible object store.
|
||||
*/
|
||||
export class Bucket {
|
||||
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string): Promise<Bucket> {
|
||||
const command = new plugins.s3.ListBucketsCommand({});
|
||||
const buckets = await smartbucketRef.s3Client.send(command);
|
||||
const buckets = await smartbucketRef.storageClient.send(command);
|
||||
const foundBucket = buckets.Buckets!.find((bucket) => bucket.Name === bucketNameArg);
|
||||
|
||||
if (foundBucket) {
|
||||
@@ -32,13 +32,13 @@ export class Bucket {
|
||||
|
||||
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||
const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName });
|
||||
await smartbucketRef.s3Client.send(command);
|
||||
await smartbucketRef.storageClient.send(command);
|
||||
return new Bucket(smartbucketRef, bucketName);
|
||||
}
|
||||
|
||||
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
|
||||
const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName });
|
||||
await smartbucketRef.s3Client.send(command);
|
||||
await smartbucketRef.storageClient.send(command);
|
||||
}
|
||||
|
||||
public smartbucketRef: SmartBucket;
|
||||
@@ -112,7 +112,7 @@ export class Bucket {
|
||||
Key: reducedPath,
|
||||
Body: optionsArg.contents,
|
||||
});
|
||||
await this.smartbucketRef.s3Client.send(command);
|
||||
await this.smartbucketRef.storageClient.send(command);
|
||||
|
||||
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
|
||||
const parsedPath = plugins.path.parse(reducedPath);
|
||||
@@ -172,7 +172,7 @@ export class Bucket {
|
||||
Bucket: this.name,
|
||||
Key: optionsArg.path,
|
||||
});
|
||||
const response = await this.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.smartbucketRef.storageClient.send(command);
|
||||
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
|
||||
|
||||
// Convert the stream to a format that supports piping
|
||||
@@ -216,7 +216,7 @@ export class Bucket {
|
||||
Bucket: this.name,
|
||||
Key: optionsArg.path,
|
||||
});
|
||||
const response = await this.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.smartbucketRef.storageClient.send(command);
|
||||
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||
|
||||
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
|
||||
@@ -272,7 +272,7 @@ export class Bucket {
|
||||
Body: optionsArg.readableStream,
|
||||
Metadata: optionsArg.nativeMetadata,
|
||||
});
|
||||
await this.smartbucketRef.s3Client.send(command);
|
||||
await this.smartbucketRef.storageClient.send(command);
|
||||
|
||||
console.log(
|
||||
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
|
||||
@@ -297,7 +297,7 @@ export class Bucket {
|
||||
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
|
||||
|
||||
// Retrieve current object information to use in copy conditions
|
||||
const currentObjInfo = await this.smartbucketRef.s3Client.send(
|
||||
const currentObjInfo = await this.smartbucketRef.storageClient.send(
|
||||
new plugins.s3.HeadObjectCommand({
|
||||
Bucket: this.name,
|
||||
Key: optionsArg.sourcePath,
|
||||
@@ -319,7 +319,7 @@ export class Bucket {
|
||||
Metadata: newNativeMetadata,
|
||||
MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
|
||||
});
|
||||
await this.smartbucketRef.s3Client.send(command);
|
||||
await this.smartbucketRef.storageClient.send(command);
|
||||
} catch (err) {
|
||||
console.error('Error updating metadata:', err);
|
||||
throw err; // rethrow to allow caller to handle
|
||||
@@ -379,7 +379,7 @@ export class Bucket {
|
||||
Bucket: this.name,
|
||||
Key: optionsArg.path,
|
||||
});
|
||||
await this.smartbucketRef.s3Client.send(command);
|
||||
await this.smartbucketRef.storageClient.send(command);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -393,7 +393,7 @@ export class Bucket {
|
||||
Bucket: this.name,
|
||||
Key: optionsArg.path,
|
||||
});
|
||||
await this.smartbucketRef.s3Client.send(command);
|
||||
await this.smartbucketRef.storageClient.send(command);
|
||||
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
@@ -411,7 +411,7 @@ export class Bucket {
|
||||
* deletes this bucket
|
||||
*/
|
||||
public async delete() {
|
||||
await this.smartbucketRef.s3Client.send(
|
||||
await this.smartbucketRef.storageClient.send(
|
||||
new plugins.s3.DeleteBucketCommand({ Bucket: this.name })
|
||||
);
|
||||
}
|
||||
@@ -422,7 +422,7 @@ export class Bucket {
|
||||
Bucket: this.name,
|
||||
Key: checkPath,
|
||||
});
|
||||
return this.smartbucketRef.s3Client.send(command);
|
||||
return this.smartbucketRef.storageClient.send(command);
|
||||
}
|
||||
|
||||
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
|
||||
@@ -432,7 +432,7 @@ export class Bucket {
|
||||
Prefix: checkPath,
|
||||
Delimiter: '/',
|
||||
});
|
||||
const { CommonPrefixes } = await this.smartbucketRef.s3Client.send(command);
|
||||
const { CommonPrefixes } = await this.smartbucketRef.storageClient.send(command);
|
||||
return !!CommonPrefixes && CommonPrefixes.length > 0;
|
||||
}
|
||||
|
||||
@@ -443,7 +443,7 @@ export class Bucket {
|
||||
Prefix: checkPath,
|
||||
Delimiter: '/',
|
||||
});
|
||||
const { Contents } = await this.smartbucketRef.s3Client.send(command);
|
||||
const { Contents } = await this.smartbucketRef.storageClient.send(command);
|
||||
return !!Contents && Contents.length > 0;
|
||||
}
|
||||
|
||||
@@ -454,7 +454,7 @@ export class Bucket {
|
||||
Key: optionsArg.path,
|
||||
Range: `bytes=0-${optionsArg.length - 1}`,
|
||||
});
|
||||
const response = await this.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.smartbucketRef.storageClient.send(command);
|
||||
const chunks: Buffer[] = [];
|
||||
const stream = response.Body as any; // SdkStreamMixin includes readable stream
|
||||
|
||||
@@ -497,7 +497,7 @@ export class Bucket {
|
||||
ContinuationToken: continuationToken,
|
||||
});
|
||||
|
||||
const response = await this.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.smartbucketRef.storageClient.send(command);
|
||||
|
||||
for (const obj of response.Contents || []) {
|
||||
if (obj.Key) yield obj.Key;
|
||||
@@ -531,7 +531,7 @@ export class Bucket {
|
||||
ContinuationToken: token,
|
||||
});
|
||||
|
||||
const response = await this.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.smartbucketRef.storageClient.send(command);
|
||||
|
||||
for (const obj of response.Contents || []) {
|
||||
if (obj.Key) subscriber.next(obj.Key);
|
||||
@@ -646,7 +646,7 @@ export class Bucket {
|
||||
|
||||
// Explicitly type the response
|
||||
const response: plugins.s3.ListObjectsV2Output =
|
||||
await this.smartbucketRef.s3Client.send(listCommand);
|
||||
await this.smartbucketRef.storageClient.send(listCommand);
|
||||
|
||||
console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
|
||||
|
||||
@@ -660,7 +660,7 @@ export class Bucket {
|
||||
},
|
||||
});
|
||||
|
||||
await this.smartbucketRef.s3Client.send(deleteCommand);
|
||||
await this.smartbucketRef.storageClient.send(deleteCommand);
|
||||
}
|
||||
|
||||
// Update continuation token and truncation status
|
||||
|
||||
@@ -135,7 +135,7 @@ export class Directory {
|
||||
Delimiter: delimiter,
|
||||
ContinuationToken: continuationToken,
|
||||
});
|
||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||
|
||||
if (response.Contents) {
|
||||
allContents.push(...response.Contents);
|
||||
@@ -213,7 +213,7 @@ export class Directory {
|
||||
Prefix: this.getBasePath(),
|
||||
Delimiter: '/',
|
||||
});
|
||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||
return response.Contents;
|
||||
}
|
||||
|
||||
@@ -222,12 +222,12 @@ export class Directory {
|
||||
*/
|
||||
public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
|
||||
/**
|
||||
* in s3 a directory does not exist if it is empty
|
||||
* in object storage a directory does not exist if it is empty
|
||||
* this option returns a directory even if it is empty
|
||||
*/
|
||||
getEmptyDirectory?: boolean;
|
||||
/**
|
||||
* in s3 a directory does not exist if it is empty
|
||||
* in object storage a directory does not exist if it is empty
|
||||
* this option creates a directory even if it is empty using a initializer file
|
||||
*/
|
||||
createWithInitializerFile?: boolean;
|
||||
|
||||
@@ -12,7 +12,7 @@ export class File {
|
||||
|
||||
/**
|
||||
* creates a file in draft mode
|
||||
* you need to call .save() to store it in s3
|
||||
* you need to call .save() to store it in object storage
|
||||
* @param optionsArg
|
||||
*/
|
||||
public static async create(optionsArg: {
|
||||
|
||||
@@ -45,7 +45,7 @@ export class ListCursor {
|
||||
ContinuationToken: this.continuationToken,
|
||||
});
|
||||
|
||||
const response = await this.bucket.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.bucket.smartbucketRef.storageClient.send(command);
|
||||
|
||||
const keys = (response.Contents || [])
|
||||
.map((obj) => obj.Key)
|
||||
|
||||
@@ -2,26 +2,28 @@
|
||||
|
||||
import * as plugins from './plugins.js';
|
||||
import { Bucket } from './classes.bucket.js';
|
||||
import { normalizeS3Descriptor } from './helpers.js';
|
||||
import { normalizeStorageDescriptor } from './helpers.js';
|
||||
|
||||
export class SmartBucket {
|
||||
public config: plugins.tsclass.storage.IS3Descriptor;
|
||||
public config: plugins.tsclass.storage.IStorageDescriptor;
|
||||
|
||||
public s3Client: plugins.s3.S3Client;
|
||||
public storageClient: plugins.s3.S3Client;
|
||||
|
||||
/** @deprecated Use storageClient instead */
|
||||
public get s3Client(): plugins.s3.S3Client {
|
||||
return this.storageClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* the constructor of SmartBucket
|
||||
*/
|
||||
/**
|
||||
* the constructor of SmartBucket
|
||||
*/
|
||||
constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
|
||||
constructor(configArg: plugins.tsclass.storage.IStorageDescriptor) {
|
||||
this.config = configArg;
|
||||
|
||||
// Use the normalizer to handle various endpoint formats
|
||||
const { normalized } = normalizeS3Descriptor(configArg);
|
||||
const { normalized } = normalizeStorageDescriptor(configArg);
|
||||
|
||||
this.s3Client = new plugins.s3.S3Client({
|
||||
this.storageClient = new plugins.s3.S3Client({
|
||||
endpoint: normalized.endpointUrl,
|
||||
region: normalized.region,
|
||||
credentials: normalized.credentials,
|
||||
@@ -47,7 +49,7 @@ export class SmartBucket {
|
||||
*/
|
||||
public async bucketExists(bucketNameArg: string): Promise<boolean> {
|
||||
const command = new plugins.s3.ListBucketsCommand({});
|
||||
const buckets = await this.s3Client.send(command);
|
||||
const buckets = await this.storageClient.send(command);
|
||||
return buckets.Buckets?.some(bucket => bucket.Name === bucketNameArg) ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import type { Bucket } from './classes.bucket.js';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
/**
|
||||
* BucketWatcher monitors an S3 bucket for changes (add/modify/delete)
|
||||
* BucketWatcher monitors a storage bucket for changes (add/modify/delete)
|
||||
* using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern.
|
||||
*
|
||||
* @example
|
||||
@@ -34,11 +34,11 @@ export class BucketWatcher extends EventEmitter {
|
||||
public readyDeferred = plugins.smartpromise.defer();
|
||||
|
||||
/** Observable for receiving change events (supports RxJS operators) */
|
||||
public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IS3ChangeEvent | interfaces.IS3ChangeEvent[]>;
|
||||
public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IStorageChangeEvent | interfaces.IStorageChangeEvent[]>;
|
||||
|
||||
// Internal subjects and state
|
||||
private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>;
|
||||
private previousState: Map<string, interfaces.IS3ObjectState>;
|
||||
private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>;
|
||||
private previousState: Map<string, interfaces.IStorageObjectState>;
|
||||
private pollIntervalId: ReturnType<typeof setInterval> | null = null;
|
||||
private isPolling = false;
|
||||
private isStopped = false;
|
||||
@@ -65,13 +65,13 @@ export class BucketWatcher extends EventEmitter {
|
||||
this.previousState = new Map();
|
||||
|
||||
// Initialize raw subject for emitting changes
|
||||
this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>();
|
||||
this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>();
|
||||
|
||||
// Configure the public observable with optional buffering
|
||||
if (this.bufferTimeMs && this.bufferTimeMs > 0) {
|
||||
this.changeSubject = this.rawSubject.pipe(
|
||||
plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs),
|
||||
plugins.smartrx.rxjs.ops.filter((events: interfaces.IS3ChangeEvent[]) => events.length > 0)
|
||||
plugins.smartrx.rxjs.ops.filter((events: interfaces.IStorageChangeEvent[]) => events.length > 0)
|
||||
);
|
||||
} else {
|
||||
this.changeSubject = this.rawSubject.asObservable();
|
||||
@@ -174,7 +174,7 @@ export class BucketWatcher extends EventEmitter {
|
||||
|
||||
try {
|
||||
// Build current state
|
||||
const currentState = new Map<string, interfaces.IS3ObjectState>();
|
||||
const currentState = new Map<string, interfaces.IStorageObjectState>();
|
||||
|
||||
for await (const obj of this.listObjectsWithMetadata()) {
|
||||
if (this.isStopped) {
|
||||
@@ -205,7 +205,7 @@ export class BucketWatcher extends EventEmitter {
|
||||
/**
|
||||
* Detect changes between current and previous state
|
||||
*/
|
||||
private detectChanges(currentState: Map<string, interfaces.IS3ObjectState>): void {
|
||||
private detectChanges(currentState: Map<string, interfaces.IStorageObjectState>): void {
|
||||
// Detect added and modified objects
|
||||
for (const [key, current] of currentState) {
|
||||
const previous = this.previousState.get(key);
|
||||
@@ -253,7 +253,7 @@ export class BucketWatcher extends EventEmitter {
|
||||
/**
|
||||
* Emit a change event via both RxJS Subject and EventEmitter
|
||||
*/
|
||||
private emitChange(event: interfaces.IS3ChangeEvent): void {
|
||||
private emitChange(event: interfaces.IStorageChangeEvent): void {
|
||||
this.rawSubject.next(event);
|
||||
this.emit('change', event);
|
||||
}
|
||||
@@ -277,7 +277,7 @@ export class BucketWatcher extends EventEmitter {
|
||||
ContinuationToken: continuationToken,
|
||||
});
|
||||
|
||||
const response = await this.bucketRef.smartbucketRef.s3Client.send(command);
|
||||
const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
|
||||
|
||||
for (const obj of response.Contents || []) {
|
||||
yield obj;
|
||||
|
||||
@@ -21,13 +21,13 @@ export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.I
|
||||
return returnPath;
|
||||
}
|
||||
|
||||
// S3 Descriptor Normalization
|
||||
export interface IS3Warning {
|
||||
// Storage Descriptor Normalization
|
||||
export interface IStorageWarning {
|
||||
code: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface INormalizedS3Config {
|
||||
export interface INormalizedStorageConfig {
|
||||
endpointUrl: string;
|
||||
host: string;
|
||||
protocol: 'http' | 'https';
|
||||
@@ -40,7 +40,7 @@ export interface INormalizedS3Config {
|
||||
forcePathStyle: boolean;
|
||||
}
|
||||
|
||||
function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IS3Warning } {
|
||||
function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IStorageWarning } {
|
||||
if (typeof value === 'boolean') return { value };
|
||||
if (typeof value === 'string') {
|
||||
const v = value.trim().toLowerCase();
|
||||
@@ -66,7 +66,7 @@ function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warni
|
||||
return { value: undefined };
|
||||
}
|
||||
|
||||
function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IS3Warning } {
|
||||
function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IStorageWarning } {
|
||||
if (port === undefined || port === null || port === '') return { value: undefined };
|
||||
const n = typeof port === 'number' ? port : Number(String(port).trim());
|
||||
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) {
|
||||
@@ -81,8 +81,8 @@ function coercePortMaybe(port: unknown): { value: number | undefined; warning?:
|
||||
return { value: n };
|
||||
}
|
||||
|
||||
function sanitizeEndpointString(raw: unknown): { value: string; warnings: IS3Warning[] } {
|
||||
const warnings: IS3Warning[] = [];
|
||||
function sanitizeEndpointString(raw: unknown): { value: string; warnings: IStorageWarning[] } {
|
||||
const warnings: IStorageWarning[] = [];
|
||||
let s = String(raw ?? '').trim();
|
||||
if (s !== String(raw ?? '')) {
|
||||
warnings.push({
|
||||
@@ -138,17 +138,17 @@ function parseEndpointHostPort(
|
||||
return { hadScheme, host, port, extras };
|
||||
}
|
||||
|
||||
export function normalizeS3Descriptor(
|
||||
input: plugins.tsclass.storage.IS3Descriptor,
|
||||
export function normalizeStorageDescriptor(
|
||||
input: plugins.tsclass.storage.IStorageDescriptor,
|
||||
logger?: { warn: (msg: string) => void }
|
||||
): { normalized: INormalizedS3Config; warnings: IS3Warning[] } {
|
||||
const warnings: IS3Warning[] = [];
|
||||
const logWarn = (w: IS3Warning) => {
|
||||
): { normalized: INormalizedStorageConfig; warnings: IStorageWarning[] } {
|
||||
const warnings: IStorageWarning[] = [];
|
||||
const logWarn = (w: IStorageWarning) => {
|
||||
warnings.push(w);
|
||||
if (logger) {
|
||||
logger.warn(`[SmartBucket S3] ${w.code}: ${w.message}`);
|
||||
logger.warn(`[SmartBucket] ${w.code}: ${w.message}`);
|
||||
} else {
|
||||
console.warn(`[SmartBucket S3] ${w.code}: ${w.message}`);
|
||||
console.warn(`[SmartBucket] ${w.code}: ${w.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -163,7 +163,7 @@ export function normalizeS3Descriptor(
|
||||
endpointSanWarnings.forEach(logWarn);
|
||||
|
||||
if (!endpointStr) {
|
||||
throw new Error('S3 endpoint is required (got empty string). Provide hostname or URL.');
|
||||
throw new Error('Storage endpoint is required (got empty string). Provide hostname or URL.');
|
||||
}
|
||||
|
||||
// Provisional protocol selection for parsing host:port forms
|
||||
|
||||
@@ -10,9 +10,9 @@ export interface IPathDecriptor {
|
||||
// ================================
|
||||
|
||||
/**
|
||||
* Internal state tracking for an S3 object
|
||||
* Internal state tracking for a storage object
|
||||
*/
|
||||
export interface IS3ObjectState {
|
||||
export interface IStorageObjectState {
|
||||
key: string;
|
||||
etag: string;
|
||||
size: number;
|
||||
@@ -22,7 +22,7 @@ export interface IS3ObjectState {
|
||||
/**
|
||||
* Change event emitted by BucketWatcher
|
||||
*/
|
||||
export interface IS3ChangeEvent {
|
||||
export interface IStorageChangeEvent {
|
||||
type: 'add' | 'modify' | 'delete';
|
||||
key: string;
|
||||
size?: number;
|
||||
@@ -54,4 +54,14 @@ export interface IBucketWatcherOptions {
|
||||
pageSize?: number;
|
||||
// Future websocket options will be added here
|
||||
// websocketUrl?: string;
|
||||
}
|
||||
}
|
||||
|
||||
// ================================
|
||||
// Deprecated aliases
|
||||
// ================================
|
||||
|
||||
/** @deprecated Use IStorageObjectState instead */
|
||||
export type IS3ObjectState = IStorageObjectState;
|
||||
|
||||
/** @deprecated Use IStorageChangeEvent instead */
|
||||
export type IS3ChangeEvent = IStorageChangeEvent;
|
||||
|
||||
Reference in New Issue
Block a user