feat(listing): Add memory-efficient listing APIs: async generator, RxJS observable, and cursor pagination; export ListCursor and Minimatch; add minimatch dependency; bump to 4.2.0

ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartbucket',
-  version: '4.1.0',
+  version: '4.3.0',
   description: 'A TypeScript library providing a cloud-agnostic interface for managing object storage with functionalities like bucket management, file and directory operations, and advanced features such as metadata handling and file locking.'
 }

ts/classes.bucket.ts

@@ -7,6 +7,7 @@ import { SmartBucket } from './classes.smartbucket.js';
 import { Directory } from './classes.directory.js';
 import { File } from './classes.file.js';
 import { Trash } from './classes.trash.js';
+import { ListCursor, type IListCursorOptions } from './classes.listcursor.js';
 
 /**
  * The bucket class exposes the basic functionality of a bucket.
@@ -469,6 +470,145 @@ export class Bucket {
     }
   }
 
+  // ==========================================
+  // Memory-Efficient Listing Methods (Phase 1)
+  // ==========================================
+
+  /**
+   * List all objects with a given prefix using async generator (memory-efficient streaming)
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @yields Object keys one at a time
+   * @example
+   * ```ts
+   * for await (const key of bucket.listAllObjects('npm/')) {
+   *   console.log(key);
+   *   if (shouldStop) break; // Early exit supported
+   * }
+   * ```
+   */
+  public async *listAllObjects(prefix: string = ''): AsyncIterableIterator<string> {
+    let continuationToken: string | undefined;
+
+    do {
+      const command = new plugins.s3.ListObjectsV2Command({
+        Bucket: this.name,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      });
+
+      const response = await this.smartbucketRef.s3Client.send(command);
+
+      for (const obj of response.Contents || []) {
+        if (obj.Key) yield obj.Key;
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+  }
+
+  /**
+   * List all objects as an RxJS Observable (for complex reactive pipelines)
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @returns Observable that emits object keys
+   * @example
+   * ```ts
+   * bucket.listAllObjectsObservable('npm/')
+   *   .pipe(
+   *     filter(key => key.endsWith('.json')),
+   *     take(100)
+   *   )
+   *   .subscribe(key => console.log(key));
+   * ```
+   */
+  public listAllObjectsObservable(prefix: string = ''): plugins.smartrx.rxjs.Observable<string> {
+    return new plugins.smartrx.rxjs.Observable<string>((subscriber) => {
+      const fetchPage = async (token?: string) => {
+        try {
+          const command = new plugins.s3.ListObjectsV2Command({
+            Bucket: this.name,
+            Prefix: prefix,
+            ContinuationToken: token,
+          });
+
+          const response = await this.smartbucketRef.s3Client.send(command);
+
+          for (const obj of response.Contents || []) {
+            if (obj.Key) subscriber.next(obj.Key);
+          }
+
+          if (response.NextContinuationToken) {
+            await fetchPage(response.NextContinuationToken);
+          } else {
+            subscriber.complete();
+          }
+        } catch (error) {
+          subscriber.error(error);
+        }
+      };
+
+      fetchPage();
+    });
+  }
+
+  /**
+   * Create a cursor for manual pagination control
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @param options - Cursor options (pageSize, etc.)
+   * @returns ListCursor instance
+   * @example
+   * ```ts
+   * const cursor = bucket.createCursor('npm/', { pageSize: 500 });
+   * while (cursor.hasMore()) {
+   *   const { keys, done } = await cursor.next();
+   *   console.log(`Processing ${keys.length} keys...`);
+   * }
+   * ```
+   */
+  public createCursor(prefix: string = '', options?: IListCursorOptions): ListCursor {
+    return new ListCursor(this, prefix, options);
+  }
+
+  // ==========================================
+  // High-Level Listing Helpers (Phase 2)
+  // ==========================================
+
+  /**
+   * Find objects matching a glob pattern (memory-efficient)
+   * @param pattern - Glob pattern (e.g., "**\/*.json", "npm/packages/*\/index.json")
+   * @yields Matching object keys
+   * @example
+   * ```ts
+   * for await (const key of bucket.findByGlob('npm/packages/*\/index.json')) {
+   *   console.log('Found package index:', key);
+   * }
+   * ```
+   */
+  public async *findByGlob(pattern: string): AsyncIterableIterator<string> {
+    const matcher = new plugins.Minimatch(pattern);
+    for await (const key of this.listAllObjects('')) {
+      if (matcher.match(key)) yield key;
+    }
+  }
+
+  /**
+   * List all objects and collect into an array (convenience method)
+   * WARNING: Loads entire result set into memory. Use listAllObjects() generator for large buckets.
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @returns Array of all object keys
+   * @example
+   * ```ts
+   * const allKeys = await bucket.listAllObjectsArray('npm/');
+   * console.log(`Found ${allKeys.length} objects`);
+   * ```
+   */
+  public async listAllObjectsArray(prefix: string = ''): Promise<string[]> {
+    const keys: string[] = [];
+    for await (const key of this.listAllObjects(prefix)) {
+      keys.push(key);
+    }
+    return keys;
+  }
+
   public async cleanAllContents(): Promise<void> {
     try {
       // Define the command type explicitly

ts/classes.listcursor.ts

@@ -0,0 +1,89 @@
+// classes.listcursor.ts
+
+import * as plugins from './plugins.js';
+import type { Bucket } from './classes.bucket.js';
+
+export interface IListCursorOptions {
+  pageSize?: number;
+}
+
+export interface IListCursorResult {
+  keys: string[];
+  done: boolean;
+}
+
+/**
+ * ListCursor provides explicit pagination control for listing objects in a bucket.
+ * Useful for UI pagination, resumable operations, and manual batch processing.
+ */
+export class ListCursor {
+  private continuationToken?: string;
+  private exhausted = false;
+  private pageSize: number;
+
+  constructor(
+    private bucket: Bucket,
+    private prefix: string,
+    options: IListCursorOptions = {}
+  ) {
+    this.pageSize = options.pageSize || 1000;
+  }
+
+  /**
+   * Fetch the next page of object keys
+   * @returns Object with keys array and done flag
+   */
+  public async next(): Promise<IListCursorResult> {
+    if (this.exhausted) {
+      return { keys: [], done: true };
+    }
+
+    const command = new plugins.s3.ListObjectsV2Command({
+      Bucket: this.bucket.name,
+      Prefix: this.prefix,
+      MaxKeys: this.pageSize,
+      ContinuationToken: this.continuationToken,
+    });
+
+    const response = await this.bucket.smartbucketRef.s3Client.send(command);
+
+    const keys = (response.Contents || [])
+      .map((obj) => obj.Key)
+      .filter((key): key is string => !!key);
+
+    this.continuationToken = response.NextContinuationToken;
+    this.exhausted = !this.continuationToken;
+
+    return { keys, done: this.exhausted };
+  }
+
+  /**
+   * Check if there are more pages to fetch
+   */
+  public hasMore(): boolean {
+    return !this.exhausted;
+  }
+
+  /**
+   * Reset the cursor to start from the beginning
+   */
+  public reset(): void {
+    this.continuationToken = undefined;
+    this.exhausted = false;
+  }
+
+  /**
+   * Get the current continuation token (for saving/restoring state)
+   */
+  public getToken(): string | undefined {
+    return this.continuationToken;
+  }
+
+  /**
+   * Set the continuation token (for resuming from a saved state)
+   */
+  public setToken(token: string | undefined): void {
+    this.continuationToken = token;
+    this.exhausted = !token;
+  }
+}

ts/index.ts

@@ -2,5 +2,6 @@ export * from './classes.smartbucket.js';
 export * from './classes.bucket.js';
 export * from './classes.directory.js';
 export * from './classes.file.js';
+export * from './classes.listcursor.js';
 export * from './classes.metadata.js';
 export * from './classes.trash.js';

ts/plugins.ts

@@ -26,7 +26,9 @@ export {
 
 // third party scope
 import * as s3 from '@aws-sdk/client-s3';
+import { Minimatch } from 'minimatch';
 
 export {
   s3,
+  Minimatch,
 }