Compare commits

..

2 Commits

15 changed files with 3599 additions and 3203 deletions

View File

@@ -1,5 +1,13 @@
# Changelog # Changelog
## 2026-03-14 - 4.5.0 - feat(storage)
generalize S3 client and watcher interfaces to storage-oriented naming with backward compatibility
- rename SmartBucket internals from s3Client to storageClient and normalize configuration using storage descriptor types
- update watcher and interface types from S3-specific names to storage-oriented equivalents while keeping deprecated aliases
- refresh tests and documentation comments to reflect S3-compatible object storage terminology
- bump build, test, AWS SDK, and related dependency versions
## 2026-01-25 - 4.4.1 - fix(tests) ## 2026-01-25 - 4.4.1 - fix(tests)
add explicit 'as string' type assertions to environment variable retrievals in tests add explicit 'as string' type assertions to environment variable retrievals in tests

3044
deno.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{ {
"name": "@push.rocks/smartbucket", "name": "@push.rocks/smartbucket",
"version": "4.4.1", "version": "4.5.0",
"description": "A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.", "description": "A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.",
"main": "dist_ts/index.js", "main": "dist_ts/index.js",
"typings": "dist_ts/index.d.ts", "typings": "dist_ts/index.d.ts",
@@ -12,23 +12,24 @@
"build": "(tsbuild tsfolders --allowimplicitany)" "build": "(tsbuild tsfolders --allowimplicitany)"
}, },
"devDependencies": { "devDependencies": {
"@git.zone/tsbuild": "^4.1.2", "@git.zone/tsbuild": "^4.3.0",
"@git.zone/tsrun": "^2.0.1", "@git.zone/tsrun": "^2.0.1",
"@git.zone/tstest": "^3.1.6", "@git.zone/tstest": "^3.3.2",
"@push.rocks/qenv": "^6.1.3", "@push.rocks/qenv": "^6.1.3",
"@push.rocks/tapbundle": "^6.0.3" "@push.rocks/tapbundle": "^6.0.3",
"@types/node": "^22.15.29"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.975.0", "@aws-sdk/client-s3": "^3.1009.0",
"@push.rocks/smartmime": "^2.0.4", "@push.rocks/smartmime": "^2.0.4",
"@push.rocks/smartpath": "^6.0.0", "@push.rocks/smartpath": "^6.0.0",
"@push.rocks/smartpromise": "^4.2.3", "@push.rocks/smartpromise": "^4.2.3",
"@push.rocks/smartrx": "^3.0.10", "@push.rocks/smartrx": "^3.0.10",
"@push.rocks/smartstream": "^3.2.5", "@push.rocks/smartstream": "^3.4.0",
"@push.rocks/smartstring": "^4.1.0", "@push.rocks/smartstring": "^4.1.0",
"@push.rocks/smartunique": "^3.0.9", "@push.rocks/smartunique": "^3.0.9",
"@tsclass/tsclass": "^9.3.0", "@tsclass/tsclass": "^9.4.0",
"minimatch": "^10.1.1" "minimatch": "^10.2.4"
}, },
"private": false, "private": false,
"files": [ "files": [

3557
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -3,7 +3,7 @@ import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as plugins from '../ts/plugins.js'; import * as plugins from '../ts/plugins.js';
import * as smartbucket from '../ts/index.js'; import * as smartbucket from '../ts/index.js';
class FakeS3Client { class FakeStorageClient {
private callIndex = 0; private callIndex = 0;
constructor(private readonly pages: Array<Partial<plugins.s3.ListObjectsV2Output>>) {} constructor(private readonly pages: Array<Partial<plugins.s3.ListObjectsV2Output>>) {}
@@ -30,7 +30,7 @@ tap.test('MetaData.hasMetaData should return false when metadata file does not e
}); });
tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => { tap.test('getSubDirectoryByName should create correct parent chain for new nested directories', async () => {
const fakeSmartbucket = { s3Client: new FakeS3Client([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket; const fakeSmartbucket = { storageClient: new FakeStorageClient([{ Contents: [], CommonPrefixes: [] }]) } as unknown as smartbucket.SmartBucket;
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket'); const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
const baseDirectory = new smartbucket.Directory(bucket, null as any, ''); const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
@@ -51,7 +51,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })), Contents: Array.from({ length: 200 }, (_, index) => ({ Key: `file-${1000 + index}` })),
IsTruncated: false, IsTruncated: false,
}; };
const fakeSmartbucket = { s3Client: new FakeS3Client([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket; const fakeSmartbucket = { storageClient: new FakeStorageClient([firstPage, secondPage]) } as unknown as smartbucket.SmartBucket;
const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket'); const bucket = new smartbucket.Bucket(fakeSmartbucket, 'test-bucket');
const baseDirectory = new smartbucket.Directory(bucket, null as any, ''); const baseDirectory = new smartbucket.Directory(bucket, null as any, '');
@@ -61,7 +61,7 @@ tap.test('listFiles should aggregate results across paginated ListObjectsV2 resp
tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => { tap.test('listDirectories should aggregate CommonPrefixes across pagination', async () => {
const fakeSmartbucket = { const fakeSmartbucket = {
s3Client: new FakeS3Client([ storageClient: new FakeStorageClient([
{ CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' }, { CommonPrefixes: [{ Prefix: 'dirA/' }], IsTruncated: true, NextContinuationToken: 'token-1' },
{ CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false }, { CommonPrefixes: [{ Prefix: 'dirB/' }], IsTruncated: false },
]), ]),

View File

@@ -2,7 +2,7 @@
import { tap, expect } from '@git.zone/tstest/tapbundle'; import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as smartbucket from '../ts/index.js'; import * as smartbucket from '../ts/index.js';
import type { IS3ChangeEvent } from '../ts/interfaces.js'; import type { IStorageChangeEvent } from '../ts/interfaces.js';
// Get test configuration // Get test configuration
import * as qenv from '@push.rocks/qenv'; import * as qenv from '@push.rocks/qenv';
@@ -57,7 +57,7 @@ tap.test('should create watcher with custom options', async () => {
// ========================== // ==========================
tap.test('should detect add events for new files', async () => { tap.test('should detect add events for new files', async () => {
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-test/', prefix: 'watcher-test/',
pollIntervalMs: 500, pollIntervalMs: 500,
@@ -94,7 +94,7 @@ tap.test('should detect add events for new files', async () => {
// ========================== // ==========================
tap.test('should detect modify events for changed files', async () => { tap.test('should detect modify events for changed files', async () => {
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-test/', prefix: 'watcher-test/',
pollIntervalMs: 500, pollIntervalMs: 500,
@@ -131,7 +131,7 @@ tap.test('should detect modify events for changed files', async () => {
// ========================== // ==========================
tap.test('should detect delete events for removed files', async () => { tap.test('should detect delete events for removed files', async () => {
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-test/', prefix: 'watcher-test/',
pollIntervalMs: 500, pollIntervalMs: 500,
@@ -174,7 +174,7 @@ tap.test('should emit initial state as add events when includeInitial is true',
contents: 'content 2', contents: 'content 2',
}); });
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-initial/', prefix: 'watcher-initial/',
pollIntervalMs: 10000, // Long interval - we only care about initial events pollIntervalMs: 10000, // Long interval - we only care about initial events
@@ -206,13 +206,13 @@ tap.test('should emit initial state as add events when includeInitial is true',
// ========================== // ==========================
tap.test('should emit events via EventEmitter pattern', async () => { tap.test('should emit events via EventEmitter pattern', async () => {
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-emitter/', prefix: 'watcher-emitter/',
pollIntervalMs: 500, pollIntervalMs: 500,
}); });
watcher.on('change', (event: IS3ChangeEvent) => { watcher.on('change', (event: IStorageChangeEvent) => {
events.push(event); events.push(event);
}); });
@@ -239,7 +239,7 @@ tap.test('should emit events via EventEmitter pattern', async () => {
// ========================== // ==========================
tap.test('should buffer events when bufferTimeMs is set', async () => { tap.test('should buffer events when bufferTimeMs is set', async () => {
const bufferedEvents: (IS3ChangeEvent | IS3ChangeEvent[])[] = []; const bufferedEvents: (IStorageChangeEvent | IStorageChangeEvent[])[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-buffer/', prefix: 'watcher-buffer/',
pollIntervalMs: 200, pollIntervalMs: 200,
@@ -327,8 +327,8 @@ tap.test('should stop gracefully with stop()', async () => {
await watcher.stop(); await watcher.stop();
// Watcher should not poll after stop // Watcher should not poll after stop
const eventsCaptured: IS3ChangeEvent[] = []; const eventsCaptured: IStorageChangeEvent[] = [];
watcher.on('change', (event: IS3ChangeEvent) => { watcher.on('change', (event: IStorageChangeEvent) => {
eventsCaptured.push(event); eventsCaptured.push(event);
}); });
@@ -362,7 +362,7 @@ tap.test('should stop gracefully with close() alias', async () => {
// ========================== // ==========================
tap.test('should only detect changes within specified prefix', async () => { tap.test('should only detect changes within specified prefix', async () => {
const events: IS3ChangeEvent[] = []; const events: IStorageChangeEvent[] = [];
const watcher = testBucket.createWatcher({ const watcher = testBucket.createWatcher({
prefix: 'watcher-prefix-a/', prefix: 'watcher-prefix-a/',
pollIntervalMs: 500, pollIntervalMs: 500,

View File

@@ -3,6 +3,6 @@
*/ */
export const commitinfo = { export const commitinfo = {
name: '@push.rocks/smartbucket', name: '@push.rocks/smartbucket',
version: '4.4.1', version: '4.5.0',
description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.' description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.'
} }

View File

@@ -13,12 +13,12 @@ import { BucketWatcher } from './classes.watcher.js';
/** /**
* The bucket class exposes the basic functionality of a bucket. * The bucket class exposes the basic functionality of a bucket.
* The functions of the bucket alone are enough to * The functions of the bucket alone are enough to
* operate in S3 basic fashion on blobs of data. * operate on blobs of data in an S3-compatible object store.
*/ */
export class Bucket { export class Bucket {
public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string): Promise<Bucket> { public static async getBucketByName(smartbucketRef: SmartBucket, bucketNameArg: string): Promise<Bucket> {
const command = new plugins.s3.ListBucketsCommand({}); const command = new plugins.s3.ListBucketsCommand({});
const buckets = await smartbucketRef.s3Client.send(command); const buckets = await smartbucketRef.storageClient.send(command);
const foundBucket = buckets.Buckets!.find((bucket) => bucket.Name === bucketNameArg); const foundBucket = buckets.Buckets!.find((bucket) => bucket.Name === bucketNameArg);
if (foundBucket) { if (foundBucket) {
@@ -32,13 +32,13 @@ export class Bucket {
public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) { public static async createBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName }); const command = new plugins.s3.CreateBucketCommand({ Bucket: bucketName });
await smartbucketRef.s3Client.send(command); await smartbucketRef.storageClient.send(command);
return new Bucket(smartbucketRef, bucketName); return new Bucket(smartbucketRef, bucketName);
} }
public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) { public static async removeBucketByName(smartbucketRef: SmartBucket, bucketName: string) {
const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName }); const command = new plugins.s3.DeleteBucketCommand({ Bucket: bucketName });
await smartbucketRef.s3Client.send(command); await smartbucketRef.storageClient.send(command);
} }
public smartbucketRef: SmartBucket; public smartbucketRef: SmartBucket;
@@ -112,7 +112,7 @@ export class Bucket {
Key: reducedPath, Key: reducedPath,
Body: optionsArg.contents, Body: optionsArg.contents,
}); });
await this.smartbucketRef.s3Client.send(command); await this.smartbucketRef.storageClient.send(command);
console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`); console.log(`Object '${reducedPath}' has been successfully stored in bucket '${this.name}'.`);
const parsedPath = plugins.path.parse(reducedPath); const parsedPath = plugins.path.parse(reducedPath);
@@ -172,7 +172,7 @@ export class Bucket {
Bucket: this.name, Bucket: this.name,
Key: optionsArg.path, Key: optionsArg.path,
}); });
const response = await this.smartbucketRef.s3Client.send(command); const response = await this.smartbucketRef.storageClient.send(command);
const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>(); const replaySubject = new plugins.smartrx.rxjs.ReplaySubject<Buffer>();
// Convert the stream to a format that supports piping // Convert the stream to a format that supports piping
@@ -216,7 +216,7 @@ export class Bucket {
Bucket: this.name, Bucket: this.name,
Key: optionsArg.path, Key: optionsArg.path,
}); });
const response = await this.smartbucketRef.s3Client.send(command); const response = await this.smartbucketRef.storageClient.send(command);
const stream = response.Body as any; // SdkStreamMixin includes readable stream const stream = response.Body as any; // SdkStreamMixin includes readable stream
const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({ const duplexStream = new plugins.smartstream.SmartDuplex<Buffer, Buffer>({
@@ -272,7 +272,7 @@ export class Bucket {
Body: optionsArg.readableStream, Body: optionsArg.readableStream,
Metadata: optionsArg.nativeMetadata, Metadata: optionsArg.nativeMetadata,
}); });
await this.smartbucketRef.s3Client.send(command); await this.smartbucketRef.storageClient.send(command);
console.log( console.log(
`Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.` `Object '${optionsArg.path}' has been successfully stored in bucket '${this.name}'.`
@@ -297,7 +297,7 @@ export class Bucket {
const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name; const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
// Retrieve current object information to use in copy conditions // Retrieve current object information to use in copy conditions
const currentObjInfo = await this.smartbucketRef.s3Client.send( const currentObjInfo = await this.smartbucketRef.storageClient.send(
new plugins.s3.HeadObjectCommand({ new plugins.s3.HeadObjectCommand({
Bucket: this.name, Bucket: this.name,
Key: optionsArg.sourcePath, Key: optionsArg.sourcePath,
@@ -319,7 +319,7 @@ export class Bucket {
Metadata: newNativeMetadata, Metadata: newNativeMetadata,
MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY', MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
}); });
await this.smartbucketRef.s3Client.send(command); await this.smartbucketRef.storageClient.send(command);
} catch (err) { } catch (err) {
console.error('Error updating metadata:', err); console.error('Error updating metadata:', err);
throw err; // rethrow to allow caller to handle throw err; // rethrow to allow caller to handle
@@ -379,7 +379,7 @@ export class Bucket {
Bucket: this.name, Bucket: this.name,
Key: optionsArg.path, Key: optionsArg.path,
}); });
await this.smartbucketRef.s3Client.send(command); await this.smartbucketRef.storageClient.send(command);
} }
/** /**
@@ -393,7 +393,7 @@ export class Bucket {
Bucket: this.name, Bucket: this.name,
Key: optionsArg.path, Key: optionsArg.path,
}); });
await this.smartbucketRef.s3Client.send(command); await this.smartbucketRef.storageClient.send(command);
console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`); console.log(`Object '${optionsArg.path}' exists in bucket '${this.name}'.`);
return true; return true;
} catch (error: any) { } catch (error: any) {
@@ -411,7 +411,7 @@ export class Bucket {
* deletes this bucket * deletes this bucket
*/ */
public async delete() { public async delete() {
await this.smartbucketRef.s3Client.send( await this.smartbucketRef.storageClient.send(
new plugins.s3.DeleteBucketCommand({ Bucket: this.name }) new plugins.s3.DeleteBucketCommand({ Bucket: this.name })
); );
} }
@@ -422,7 +422,7 @@ export class Bucket {
Bucket: this.name, Bucket: this.name,
Key: checkPath, Key: checkPath,
}); });
return this.smartbucketRef.s3Client.send(command); return this.smartbucketRef.storageClient.send(command);
} }
public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> { public async isDirectory(pathDescriptor: interfaces.IPathDecriptor): Promise<boolean> {
@@ -432,7 +432,7 @@ export class Bucket {
Prefix: checkPath, Prefix: checkPath,
Delimiter: '/', Delimiter: '/',
}); });
const { CommonPrefixes } = await this.smartbucketRef.s3Client.send(command); const { CommonPrefixes } = await this.smartbucketRef.storageClient.send(command);
return !!CommonPrefixes && CommonPrefixes.length > 0; return !!CommonPrefixes && CommonPrefixes.length > 0;
} }
@@ -443,7 +443,7 @@ export class Bucket {
Prefix: checkPath, Prefix: checkPath,
Delimiter: '/', Delimiter: '/',
}); });
const { Contents } = await this.smartbucketRef.s3Client.send(command); const { Contents } = await this.smartbucketRef.storageClient.send(command);
return !!Contents && Contents.length > 0; return !!Contents && Contents.length > 0;
} }
@@ -454,7 +454,7 @@ export class Bucket {
Key: optionsArg.path, Key: optionsArg.path,
Range: `bytes=0-${optionsArg.length - 1}`, Range: `bytes=0-${optionsArg.length - 1}`,
}); });
const response = await this.smartbucketRef.s3Client.send(command); const response = await this.smartbucketRef.storageClient.send(command);
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
const stream = response.Body as any; // SdkStreamMixin includes readable stream const stream = response.Body as any; // SdkStreamMixin includes readable stream
@@ -497,7 +497,7 @@ export class Bucket {
ContinuationToken: continuationToken, ContinuationToken: continuationToken,
}); });
const response = await this.smartbucketRef.s3Client.send(command); const response = await this.smartbucketRef.storageClient.send(command);
for (const obj of response.Contents || []) { for (const obj of response.Contents || []) {
if (obj.Key) yield obj.Key; if (obj.Key) yield obj.Key;
@@ -531,7 +531,7 @@ export class Bucket {
ContinuationToken: token, ContinuationToken: token,
}); });
const response = await this.smartbucketRef.s3Client.send(command); const response = await this.smartbucketRef.storageClient.send(command);
for (const obj of response.Contents || []) { for (const obj of response.Contents || []) {
if (obj.Key) subscriber.next(obj.Key); if (obj.Key) subscriber.next(obj.Key);
@@ -646,7 +646,7 @@ export class Bucket {
// Explicitly type the response // Explicitly type the response
const response: plugins.s3.ListObjectsV2Output = const response: plugins.s3.ListObjectsV2Output =
await this.smartbucketRef.s3Client.send(listCommand); await this.smartbucketRef.storageClient.send(listCommand);
console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`); console.log(`Cleaning contents of bucket '${this.name}': Now deleting ${response.Contents?.length} items...`);
@@ -660,7 +660,7 @@ export class Bucket {
}, },
}); });
await this.smartbucketRef.s3Client.send(deleteCommand); await this.smartbucketRef.storageClient.send(deleteCommand);
} }
// Update continuation token and truncation status // Update continuation token and truncation status

View File

@@ -135,7 +135,7 @@ export class Directory {
Delimiter: delimiter, Delimiter: delimiter,
ContinuationToken: continuationToken, ContinuationToken: continuationToken,
}); });
const response = await this.bucketRef.smartbucketRef.s3Client.send(command); const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
if (response.Contents) { if (response.Contents) {
allContents.push(...response.Contents); allContents.push(...response.Contents);
@@ -213,7 +213,7 @@ export class Directory {
Prefix: this.getBasePath(), Prefix: this.getBasePath(),
Delimiter: '/', Delimiter: '/',
}); });
const response = await this.bucketRef.smartbucketRef.s3Client.send(command); const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
return response.Contents; return response.Contents;
} }
@@ -222,12 +222,12 @@ export class Directory {
*/ */
public async getSubDirectoryByName(dirNameArg: string, optionsArg: { public async getSubDirectoryByName(dirNameArg: string, optionsArg: {
/** /**
* in s3 a directory does not exist if it is empty * in object storage a directory does not exist if it is empty
* this option returns a directory even if it is empty * this option returns a directory even if it is empty
*/ */
getEmptyDirectory?: boolean; getEmptyDirectory?: boolean;
/** /**
* in s3 a directory does not exist if it is empty * in object storage a directory does not exist if it is empty
* this option creates a directory even if it is empty using a initializer file * this option creates a directory even if it is empty using a initializer file
*/ */
createWithInitializerFile?: boolean; createWithInitializerFile?: boolean;

View File

@@ -12,7 +12,7 @@ export class File {
/** /**
* creates a file in draft mode * creates a file in draft mode
* you need to call .save() to store it in s3 * you need to call .save() to store it in object storage
* @param optionsArg * @param optionsArg
*/ */
public static async create(optionsArg: { public static async create(optionsArg: {

View File

@@ -45,7 +45,7 @@ export class ListCursor {
ContinuationToken: this.continuationToken, ContinuationToken: this.continuationToken,
}); });
const response = await this.bucket.smartbucketRef.s3Client.send(command); const response = await this.bucket.smartbucketRef.storageClient.send(command);
const keys = (response.Contents || []) const keys = (response.Contents || [])
.map((obj) => obj.Key) .map((obj) => obj.Key)

View File

@@ -2,26 +2,28 @@
import * as plugins from './plugins.js'; import * as plugins from './plugins.js';
import { Bucket } from './classes.bucket.js'; import { Bucket } from './classes.bucket.js';
import { normalizeS3Descriptor } from './helpers.js'; import { normalizeStorageDescriptor } from './helpers.js';
export class SmartBucket { export class SmartBucket {
public config: plugins.tsclass.storage.IS3Descriptor; public config: plugins.tsclass.storage.IStorageDescriptor;
public s3Client: plugins.s3.S3Client; public storageClient: plugins.s3.S3Client;
/** @deprecated Use storageClient instead */
public get s3Client(): plugins.s3.S3Client {
return this.storageClient;
}
/** /**
* the constructor of SmartBucket * the constructor of SmartBucket
*/ */
/** constructor(configArg: plugins.tsclass.storage.IStorageDescriptor) {
* the constructor of SmartBucket
*/
constructor(configArg: plugins.tsclass.storage.IS3Descriptor) {
this.config = configArg; this.config = configArg;
// Use the normalizer to handle various endpoint formats // Use the normalizer to handle various endpoint formats
const { normalized } = normalizeS3Descriptor(configArg); const { normalized } = normalizeStorageDescriptor(configArg);
this.s3Client = new plugins.s3.S3Client({ this.storageClient = new plugins.s3.S3Client({
endpoint: normalized.endpointUrl, endpoint: normalized.endpointUrl,
region: normalized.region, region: normalized.region,
credentials: normalized.credentials, credentials: normalized.credentials,
@@ -47,7 +49,7 @@ export class SmartBucket {
*/ */
public async bucketExists(bucketNameArg: string): Promise<boolean> { public async bucketExists(bucketNameArg: string): Promise<boolean> {
const command = new plugins.s3.ListBucketsCommand({}); const command = new plugins.s3.ListBucketsCommand({});
const buckets = await this.s3Client.send(command); const buckets = await this.storageClient.send(command);
return buckets.Buckets?.some(bucket => bucket.Name === bucketNameArg) ?? false; return buckets.Buckets?.some(bucket => bucket.Name === bucketNameArg) ?? false;
} }
} }

View File

@@ -6,7 +6,7 @@ import type { Bucket } from './classes.bucket.js';
import { EventEmitter } from 'node:events'; import { EventEmitter } from 'node:events';
/** /**
* BucketWatcher monitors an S3 bucket for changes (add/modify/delete) * BucketWatcher monitors a storage bucket for changes (add/modify/delete)
* using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern. * using a polling-based approach. Designed to follow the SmartdataDbWatcher pattern.
* *
* @example * @example
@@ -34,11 +34,11 @@ export class BucketWatcher extends EventEmitter {
public readyDeferred = plugins.smartpromise.defer(); public readyDeferred = plugins.smartpromise.defer();
/** Observable for receiving change events (supports RxJS operators) */ /** Observable for receiving change events (supports RxJS operators) */
public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IS3ChangeEvent | interfaces.IS3ChangeEvent[]>; public changeSubject: plugins.smartrx.rxjs.Observable<interfaces.IStorageChangeEvent | interfaces.IStorageChangeEvent[]>;
// Internal subjects and state // Internal subjects and state
private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>; private rawSubject: plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>;
private previousState: Map<string, interfaces.IS3ObjectState>; private previousState: Map<string, interfaces.IStorageObjectState>;
private pollIntervalId: ReturnType<typeof setInterval> | null = null; private pollIntervalId: ReturnType<typeof setInterval> | null = null;
private isPolling = false; private isPolling = false;
private isStopped = false; private isStopped = false;
@@ -65,13 +65,13 @@ export class BucketWatcher extends EventEmitter {
this.previousState = new Map(); this.previousState = new Map();
// Initialize raw subject for emitting changes // Initialize raw subject for emitting changes
this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IS3ChangeEvent>(); this.rawSubject = new plugins.smartrx.rxjs.Subject<interfaces.IStorageChangeEvent>();
// Configure the public observable with optional buffering // Configure the public observable with optional buffering
if (this.bufferTimeMs && this.bufferTimeMs > 0) { if (this.bufferTimeMs && this.bufferTimeMs > 0) {
this.changeSubject = this.rawSubject.pipe( this.changeSubject = this.rawSubject.pipe(
plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs), plugins.smartrx.rxjs.ops.bufferTime(this.bufferTimeMs),
plugins.smartrx.rxjs.ops.filter((events: interfaces.IS3ChangeEvent[]) => events.length > 0) plugins.smartrx.rxjs.ops.filter((events: interfaces.IStorageChangeEvent[]) => events.length > 0)
); );
} else { } else {
this.changeSubject = this.rawSubject.asObservable(); this.changeSubject = this.rawSubject.asObservable();
@@ -174,7 +174,7 @@ export class BucketWatcher extends EventEmitter {
try { try {
// Build current state // Build current state
const currentState = new Map<string, interfaces.IS3ObjectState>(); const currentState = new Map<string, interfaces.IStorageObjectState>();
for await (const obj of this.listObjectsWithMetadata()) { for await (const obj of this.listObjectsWithMetadata()) {
if (this.isStopped) { if (this.isStopped) {
@@ -205,7 +205,7 @@ export class BucketWatcher extends EventEmitter {
/** /**
* Detect changes between current and previous state * Detect changes between current and previous state
*/ */
private detectChanges(currentState: Map<string, interfaces.IS3ObjectState>): void { private detectChanges(currentState: Map<string, interfaces.IStorageObjectState>): void {
// Detect added and modified objects // Detect added and modified objects
for (const [key, current] of currentState) { for (const [key, current] of currentState) {
const previous = this.previousState.get(key); const previous = this.previousState.get(key);
@@ -253,7 +253,7 @@ export class BucketWatcher extends EventEmitter {
/** /**
* Emit a change event via both RxJS Subject and EventEmitter * Emit a change event via both RxJS Subject and EventEmitter
*/ */
private emitChange(event: interfaces.IS3ChangeEvent): void { private emitChange(event: interfaces.IStorageChangeEvent): void {
this.rawSubject.next(event); this.rawSubject.next(event);
this.emit('change', event); this.emit('change', event);
} }
@@ -277,7 +277,7 @@ export class BucketWatcher extends EventEmitter {
ContinuationToken: continuationToken, ContinuationToken: continuationToken,
}); });
const response = await this.bucketRef.smartbucketRef.s3Client.send(command); const response = await this.bucketRef.smartbucketRef.storageClient.send(command);
for (const obj of response.Contents || []) { for (const obj of response.Contents || []) {
yield obj; yield obj;

View File

@@ -21,13 +21,13 @@ export const reducePathDescriptorToPath = async (pathDescriptorArg: interfaces.I
return returnPath; return returnPath;
} }
// S3 Descriptor Normalization // Storage Descriptor Normalization
export interface IS3Warning { export interface IStorageWarning {
code: string; code: string;
message: string; message: string;
} }
export interface INormalizedS3Config { export interface INormalizedStorageConfig {
endpointUrl: string; endpointUrl: string;
host: string; host: string;
protocol: 'http' | 'https'; protocol: 'http' | 'https';
@@ -40,7 +40,7 @@ export interface INormalizedS3Config {
forcePathStyle: boolean; forcePathStyle: boolean;
} }
function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IS3Warning } { function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warning?: IStorageWarning } {
if (typeof value === 'boolean') return { value }; if (typeof value === 'boolean') return { value };
if (typeof value === 'string') { if (typeof value === 'string') {
const v = value.trim().toLowerCase(); const v = value.trim().toLowerCase();
@@ -66,7 +66,7 @@ function coerceBooleanMaybe(value: unknown): { value: boolean | undefined; warni
return { value: undefined }; return { value: undefined };
} }
function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IS3Warning } { function coercePortMaybe(port: unknown): { value: number | undefined; warning?: IStorageWarning } {
if (port === undefined || port === null || port === '') return { value: undefined }; if (port === undefined || port === null || port === '') return { value: undefined };
const n = typeof port === 'number' ? port : Number(String(port).trim()); const n = typeof port === 'number' ? port : Number(String(port).trim());
if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) { if (!Number.isFinite(n) || !Number.isInteger(n) || n <= 0 || n > 65535) {
@@ -81,8 +81,8 @@ function coercePortMaybe(port: unknown): { value: number | undefined; warning?:
return { value: n }; return { value: n };
} }
function sanitizeEndpointString(raw: unknown): { value: string; warnings: IS3Warning[] } { function sanitizeEndpointString(raw: unknown): { value: string; warnings: IStorageWarning[] } {
const warnings: IS3Warning[] = []; const warnings: IStorageWarning[] = [];
let s = String(raw ?? '').trim(); let s = String(raw ?? '').trim();
if (s !== String(raw ?? '')) { if (s !== String(raw ?? '')) {
warnings.push({ warnings.push({
@@ -138,17 +138,17 @@ function parseEndpointHostPort(
return { hadScheme, host, port, extras }; return { hadScheme, host, port, extras };
} }
export function normalizeS3Descriptor( export function normalizeStorageDescriptor(
input: plugins.tsclass.storage.IS3Descriptor, input: plugins.tsclass.storage.IStorageDescriptor,
logger?: { warn: (msg: string) => void } logger?: { warn: (msg: string) => void }
): { normalized: INormalizedS3Config; warnings: IS3Warning[] } { ): { normalized: INormalizedStorageConfig; warnings: IStorageWarning[] } {
const warnings: IS3Warning[] = []; const warnings: IStorageWarning[] = [];
const logWarn = (w: IS3Warning) => { const logWarn = (w: IStorageWarning) => {
warnings.push(w); warnings.push(w);
if (logger) { if (logger) {
logger.warn(`[SmartBucket S3] ${w.code}: ${w.message}`); logger.warn(`[SmartBucket] ${w.code}: ${w.message}`);
} else { } else {
console.warn(`[SmartBucket S3] ${w.code}: ${w.message}`); console.warn(`[SmartBucket] ${w.code}: ${w.message}`);
} }
}; };
@@ -163,7 +163,7 @@ export function normalizeS3Descriptor(
endpointSanWarnings.forEach(logWarn); endpointSanWarnings.forEach(logWarn);
if (!endpointStr) { if (!endpointStr) {
throw new Error('S3 endpoint is required (got empty string). Provide hostname or URL.'); throw new Error('Storage endpoint is required (got empty string). Provide hostname or URL.');
} }
// Provisional protocol selection for parsing host:port forms // Provisional protocol selection for parsing host:port forms

View File

@@ -10,9 +10,9 @@ export interface IPathDecriptor {
// ================================ // ================================
/** /**
* Internal state tracking for an S3 object * Internal state tracking for a storage object
*/ */
export interface IS3ObjectState { export interface IStorageObjectState {
key: string; key: string;
etag: string; etag: string;
size: number; size: number;
@@ -22,7 +22,7 @@ export interface IS3ObjectState {
/** /**
* Change event emitted by BucketWatcher * Change event emitted by BucketWatcher
*/ */
export interface IS3ChangeEvent { export interface IStorageChangeEvent {
type: 'add' | 'modify' | 'delete'; type: 'add' | 'modify' | 'delete';
key: string; key: string;
size?: number; size?: number;
@@ -54,4 +54,14 @@ export interface IBucketWatcherOptions {
pageSize?: number; pageSize?: number;
// Future websocket options will be added here // Future websocket options will be added here
// websocketUrl?: string; // websocketUrl?: string;
} }
// ================================
// Deprecated aliases
// ================================
/** @deprecated Use IStorageObjectState instead */
export type IS3ObjectState = IStorageObjectState;
/** @deprecated Use IStorageChangeEvent instead */
export type IS3ChangeEvent = IStorageChangeEvent;