feat(listing): Add memory-efficient listing APIs: async generator, RxJS observable, and cursor pagination; export ListCursor and Minimatch; add minimatch dependency; bump to 4.2.0

ts/classes.bucket.ts

@@ -7,6 +7,7 @@ import { SmartBucket } from './classes.smartbucket.js';
 import { Directory } from './classes.directory.js';
 import { File } from './classes.file.js';
 import { Trash } from './classes.trash.js';
+import { ListCursor, type IListCursorOptions } from './classes.listcursor.js';
 
 /**
  * The bucket class exposes the basic functionality of a bucket.
@@ -469,6 +470,145 @@ export class Bucket {
     }
   }
 
+  // ==========================================
+  // Memory-Efficient Listing Methods (Phase 1)
+  // ==========================================
+
+  /**
+   * List all objects with a given prefix using async generator (memory-efficient streaming)
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @yields Object keys one at a time
+   * @example
+   * ```ts
+   * for await (const key of bucket.listAllObjects('npm/')) {
+   *   console.log(key);
+   *   if (shouldStop) break; // Early exit supported
+   * }
+   * ```
+   */
+  public async *listAllObjects(prefix: string = ''): AsyncIterableIterator<string> {
+    let continuationToken: string | undefined;
+
+    do {
+      const command = new plugins.s3.ListObjectsV2Command({
+        Bucket: this.name,
+        Prefix: prefix,
+        ContinuationToken: continuationToken,
+      });
+
+      const response = await this.smartbucketRef.s3Client.send(command);
+
+      for (const obj of response.Contents || []) {
+        if (obj.Key) yield obj.Key;
+      }
+
+      continuationToken = response.NextContinuationToken;
+    } while (continuationToken);
+  }
+
+  /**
+   * List all objects as an RxJS Observable (for complex reactive pipelines)
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @returns Observable that emits object keys
+   * @example
+   * ```ts
+   * bucket.listAllObjectsObservable('npm/')
+   *   .pipe(
+   *     filter(key => key.endsWith('.json')),
+   *     take(100)
+   *   )
+   *   .subscribe(key => console.log(key));
+   * ```
+   */
+  public listAllObjectsObservable(prefix: string = ''): plugins.smartrx.rxjs.Observable<string> {
+    return new plugins.smartrx.rxjs.Observable<string>((subscriber) => {
+      const fetchPage = async (token?: string) => {
+        try {
+          const command = new plugins.s3.ListObjectsV2Command({
+            Bucket: this.name,
+            Prefix: prefix,
+            ContinuationToken: token,
+          });
+
+          const response = await this.smartbucketRef.s3Client.send(command);
+
+          for (const obj of response.Contents || []) {
+            if (obj.Key) subscriber.next(obj.Key);
+          }
+
+          if (response.NextContinuationToken) {
+            await fetchPage(response.NextContinuationToken);
+          } else {
+            subscriber.complete();
+          }
+        } catch (error) {
+          subscriber.error(error);
+        }
+      };
+
+      fetchPage();
+    });
+  }
+
+  /**
+   * Create a cursor for manual pagination control
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @param options - Cursor options (pageSize, etc.)
+   * @returns ListCursor instance
+   * @example
+   * ```ts
+   * const cursor = bucket.createCursor('npm/', { pageSize: 500 });
+   * while (cursor.hasMore()) {
+   *   const { keys, done } = await cursor.next();
+   *   console.log(`Processing ${keys.length} keys...`);
+   * }
+   * ```
+   */
+  public createCursor(prefix: string = '', options?: IListCursorOptions): ListCursor {
+    return new ListCursor(this, prefix, options);
+  }
+
+  // ==========================================
+  // High-Level Listing Helpers (Phase 2)
+  // ==========================================
+
+  /**
+   * Find objects matching a glob pattern (memory-efficient)
+   * @param pattern - Glob pattern (e.g., "**\/*.json", "npm/packages/*\/index.json")
+   * @yields Matching object keys
+   * @example
+   * ```ts
+   * for await (const key of bucket.findByGlob('npm/packages/*\/index.json')) {
+   *   console.log('Found package index:', key);
+   * }
+   * ```
+   */
+  public async *findByGlob(pattern: string): AsyncIterableIterator<string> {
+    const matcher = new plugins.Minimatch(pattern);
+    for await (const key of this.listAllObjects('')) {
+      if (matcher.match(key)) yield key;
+    }
+  }
+
+  /**
+   * List all objects and collect into an array (convenience method)
+   * WARNING: Loads entire result set into memory. Use listAllObjects() generator for large buckets.
+   * @param prefix - Optional prefix to filter objects (default: '' for all objects)
+   * @returns Array of all object keys
+   * @example
+   * ```ts
+   * const allKeys = await bucket.listAllObjectsArray('npm/');
+   * console.log(`Found ${allKeys.length} objects`);
+   * ```
+   */
+  public async listAllObjectsArray(prefix: string = ''): Promise<string[]> {
+    const keys: string[] = [];
+    for await (const key of this.listAllObjects(prefix)) {
+      keys.push(key);
+    }
+    return keys;
+  }
+
   public async cleanAllContents(): Promise<void> {
     try {
       // Define the command type explicitly