feat(listing): Add memory-efficient listing APIs: async generator, RxJS observable, and cursor pagination; export ListCursor and Minimatch; add minimatch dependency; bump to 4.2.0
This commit is contained in:
298
test/test.listing.node+deno.ts
Normal file
298
test/test.listing.node+deno.ts
Normal file
@@ -0,0 +1,298 @@
|
||||
// test.listing.node+deno.ts - Tests for memory-efficient listing methods
|
||||
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as smartbucket from '../ts/index.js';
|
||||
|
||||
// Get test configuration
|
||||
import * as qenv from '@push.rocks/qenv';
|
||||
const testQenv = new qenv.Qenv('./', './.nogit/');
|
||||
|
||||
// Test bucket reference
|
||||
let testBucket: smartbucket.Bucket;
|
||||
let testSmartbucket: smartbucket.SmartBucket;
|
||||
|
||||
// Setup: Create test bucket and populate with test data
|
||||
tap.test('should create valid smartbucket and bucket', async () => {
|
||||
testSmartbucket = new smartbucket.SmartBucket({
|
||||
accessKey: await testQenv.getEnvVarOnDemand('S3_ACCESSKEY'),
|
||||
accessSecret: await testQenv.getEnvVarOnDemand('S3_SECRETKEY'),
|
||||
endpoint: await testQenv.getEnvVarOnDemand('S3_ENDPOINT'),
|
||||
port: parseInt(await testQenv.getEnvVarOnDemand('S3_PORT')),
|
||||
useSsl: false,
|
||||
});
|
||||
|
||||
testBucket = await smartbucket.Bucket.getBucketByName(
|
||||
testSmartbucket,
|
||||
await testQenv.getEnvVarOnDemand('S3_BUCKET')
|
||||
);
|
||||
expect(testBucket).toBeInstanceOf(smartbucket.Bucket);
|
||||
});
|
||||
|
||||
tap.test('should clean bucket and create test data for listing tests', async () => {
|
||||
// Clean bucket first
|
||||
await testBucket.cleanAllContents();
|
||||
|
||||
// Create test structure:
|
||||
// npm/packages/foo/index.json
|
||||
// npm/packages/foo/1.0.0.tgz
|
||||
// npm/packages/bar/index.json
|
||||
// npm/packages/bar/2.0.0.tgz
|
||||
// oci/blobs/sha256-abc.tar
|
||||
// oci/blobs/sha256-def.tar
|
||||
// oci/manifests/latest.json
|
||||
// docs/readme.md
|
||||
// docs/api.md
|
||||
|
||||
const testFiles = [
|
||||
'npm/packages/foo/index.json',
|
||||
'npm/packages/foo/1.0.0.tgz',
|
||||
'npm/packages/bar/index.json',
|
||||
'npm/packages/bar/2.0.0.tgz',
|
||||
'oci/blobs/sha256-abc.tar',
|
||||
'oci/blobs/sha256-def.tar',
|
||||
'oci/manifests/latest.json',
|
||||
'docs/readme.md',
|
||||
'docs/api.md',
|
||||
];
|
||||
|
||||
for (const filePath of testFiles) {
|
||||
await testBucket.fastPut({
|
||||
path: filePath,
|
||||
contents: `test content for ${filePath}`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// Async Generator Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('listAllObjects should iterate all objects with prefix', async () => {
|
||||
const keys: string[] = [];
|
||||
for await (const key of testBucket.listAllObjects('npm/')) {
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
expect(keys.length).toEqual(4);
|
||||
expect(keys).toContain('npm/packages/foo/index.json');
|
||||
expect(keys).toContain('npm/packages/bar/2.0.0.tgz');
|
||||
});
|
||||
|
||||
tap.test('listAllObjects should support early termination', async () => {
|
||||
let count = 0;
|
||||
for await (const key of testBucket.listAllObjects('')) {
|
||||
count++;
|
||||
if (count >= 3) break; // Early exit
|
||||
}
|
||||
|
||||
expect(count).toEqual(3);
|
||||
});
|
||||
|
||||
tap.test('listAllObjects without prefix should list all objects', async () => {
|
||||
const keys: string[] = [];
|
||||
for await (const key of testBucket.listAllObjects()) {
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
expect(keys.length).toBeGreaterThanOrEqual(9);
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// Observable Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('listAllObjectsObservable should emit all objects', async () => {
|
||||
const keys: string[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
testBucket.listAllObjectsObservable('oci/')
|
||||
.subscribe({
|
||||
next: (key) => keys.push(key),
|
||||
error: (err) => reject(err),
|
||||
complete: () => resolve(),
|
||||
});
|
||||
});
|
||||
|
||||
expect(keys.length).toEqual(3);
|
||||
expect(keys).toContain('oci/blobs/sha256-abc.tar');
|
||||
expect(keys).toContain('oci/manifests/latest.json');
|
||||
});
|
||||
|
||||
tap.test('listAllObjectsObservable should support RxJS operators', async () => {
|
||||
const jsonFiles: string[] = [];
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
testBucket.listAllObjectsObservable('npm/')
|
||||
.subscribe({
|
||||
next: (key: string) => {
|
||||
if (key.endsWith('.json')) {
|
||||
jsonFiles.push(key);
|
||||
}
|
||||
},
|
||||
error: (err: any) => reject(err),
|
||||
complete: () => resolve(),
|
||||
});
|
||||
});
|
||||
|
||||
expect(jsonFiles.length).toEqual(2);
|
||||
expect(jsonFiles.every((k) => k.endsWith('.json'))).toBeTrue();
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// Cursor Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('createCursor should allow manual pagination', async () => {
|
||||
const cursor = testBucket.createCursor('npm/', { pageSize: 2 });
|
||||
|
||||
// First page
|
||||
const page1 = await cursor.next();
|
||||
expect(page1.keys.length).toEqual(2);
|
||||
expect(page1.done).toBeFalse();
|
||||
|
||||
// Second page
|
||||
const page2 = await cursor.next();
|
||||
expect(page2.keys.length).toEqual(2);
|
||||
expect(page2.done).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('cursor.hasMore() should accurately track state', async () => {
|
||||
const cursor = testBucket.createCursor('docs/', { pageSize: 10 });
|
||||
|
||||
expect(cursor.hasMore()).toBeTrue();
|
||||
|
||||
await cursor.next(); // Should get all docs files
|
||||
|
||||
expect(cursor.hasMore()).toBeFalse();
|
||||
});
|
||||
|
||||
tap.test('cursor.reset() should allow re-iteration', async () => {
|
||||
const cursor = testBucket.createCursor('docs/');
|
||||
|
||||
const firstRun = await cursor.next();
|
||||
expect(firstRun.keys.length).toBeGreaterThan(0);
|
||||
|
||||
cursor.reset();
|
||||
expect(cursor.hasMore()).toBeTrue();
|
||||
|
||||
const secondRun = await cursor.next();
|
||||
expect(secondRun.keys).toEqual(firstRun.keys);
|
||||
});
|
||||
|
||||
tap.test('cursor should support save/restore with token', async () => {
|
||||
const cursor1 = testBucket.createCursor('npm/', { pageSize: 2 });
|
||||
|
||||
await cursor1.next(); // Advance cursor
|
||||
const token = cursor1.getToken();
|
||||
expect(token).toBeDefined();
|
||||
|
||||
// Create new cursor and restore state
|
||||
const cursor2 = testBucket.createCursor('npm/', { pageSize: 2 });
|
||||
cursor2.setToken(token);
|
||||
|
||||
const page = await cursor2.next();
|
||||
expect(page.keys.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// findByGlob Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('findByGlob should match simple patterns', async () => {
|
||||
const matches: string[] = [];
|
||||
for await (const key of testBucket.findByGlob('**/*.json')) {
|
||||
matches.push(key);
|
||||
}
|
||||
|
||||
expect(matches.length).toEqual(3); // foo/index.json, bar/index.json, latest.json
|
||||
expect(matches.every((k) => k.endsWith('.json'))).toBeTrue();
|
||||
});
|
||||
|
||||
tap.test('findByGlob should match specific path patterns', async () => {
|
||||
const matches: string[] = [];
|
||||
for await (const key of testBucket.findByGlob('npm/packages/*/index.json')) {
|
||||
matches.push(key);
|
||||
}
|
||||
|
||||
expect(matches.length).toEqual(2);
|
||||
expect(matches).toContain('npm/packages/foo/index.json');
|
||||
expect(matches).toContain('npm/packages/bar/index.json');
|
||||
});
|
||||
|
||||
tap.test('findByGlob should match wildcard patterns', async () => {
|
||||
const matches: string[] = [];
|
||||
for await (const key of testBucket.findByGlob('oci/blobs/*')) {
|
||||
matches.push(key);
|
||||
}
|
||||
|
||||
expect(matches.length).toEqual(2);
|
||||
expect(matches.every((k) => k.startsWith('oci/blobs/'))).toBeTrue();
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// listAllObjectsArray Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('listAllObjectsArray should collect all keys into array', async () => {
|
||||
const keys = await testBucket.listAllObjectsArray('docs/');
|
||||
|
||||
expect(Array.isArray(keys)).toBeTrue();
|
||||
expect(keys.length).toEqual(2);
|
||||
expect(keys).toContain('docs/readme.md');
|
||||
expect(keys).toContain('docs/api.md');
|
||||
});
|
||||
|
||||
tap.test('listAllObjectsArray without prefix should return all objects', async () => {
|
||||
const keys = await testBucket.listAllObjectsArray();
|
||||
|
||||
expect(keys.length).toBeGreaterThanOrEqual(9);
|
||||
});
|
||||
|
||||
// ==========================
|
||||
// Performance/Edge Case Tests
|
||||
// ==========================
|
||||
|
||||
tap.test('should handle empty prefix results gracefully', async () => {
|
||||
const keys: string[] = [];
|
||||
for await (const key of testBucket.listAllObjects('nonexistent/')) {
|
||||
keys.push(key);
|
||||
}
|
||||
|
||||
expect(keys.length).toEqual(0);
|
||||
});
|
||||
|
||||
tap.test('cursor should handle empty results', async () => {
|
||||
const cursor = testBucket.createCursor('nonexistent/');
|
||||
const result = await cursor.next();
|
||||
|
||||
expect(result.keys.length).toEqual(0);
|
||||
expect(result.done).toBeTrue();
|
||||
expect(cursor.hasMore()).toBeFalse();
|
||||
});
|
||||
|
||||
tap.test('observable should complete immediately on empty results', async () => {
|
||||
let completed = false;
|
||||
let count = 0;
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
testBucket.listAllObjectsObservable('nonexistent/')
|
||||
.subscribe({
|
||||
next: () => count++,
|
||||
error: (err) => reject(err),
|
||||
complete: () => {
|
||||
completed = true;
|
||||
resolve();
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
expect(count).toEqual(0);
|
||||
expect(completed).toBeTrue();
|
||||
});
|
||||
|
||||
// Cleanup
|
||||
tap.test('should clean up test data', async () => {
|
||||
await testBucket.cleanAllContents();
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -12,13 +12,16 @@ let baseDirectory: smartbucket.Directory;
|
||||
tap.test('should create a valid smartbucket', async () => {
|
||||
testSmartbucket = new smartbucket.SmartBucket({
|
||||
accessKey: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSKEY'),
|
||||
accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSSECRET'),
|
||||
accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_SECRETKEY'),
|
||||
endpoint: await testQenv.getEnvVarOnDemandStrict('S3_ENDPOINT'),
|
||||
port: parseInt(await testQenv.getEnvVarOnDemandStrict('S3_PORT')),
|
||||
useSsl: false,
|
||||
});
|
||||
expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
|
||||
myBucket = await testSmartbucket.getBucketByName(await testQenv.getEnvVarOnDemandStrict('S3_BUCKET'),);
|
||||
const bucketName = await testQenv.getEnvVarOnDemandStrict('S3_BUCKET');
|
||||
myBucket = await testSmartbucket.getBucketByName(bucketName);
|
||||
expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
|
||||
expect(myBucket.name).toEqual('test-pushrocks-smartbucket');
|
||||
expect(myBucket.name).toEqual(bucketName);
|
||||
});
|
||||
|
||||
tap.test('should clean all contents', async () => {
|
||||
|
||||
@@ -13,13 +13,15 @@ let baseDirectory: smartbucket.Directory;
|
||||
tap.test('should create a valid smartbucket', async () => {
|
||||
testSmartbucket = new smartbucket.SmartBucket({
|
||||
accessKey: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSKEY'),
|
||||
accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_ACCESSSECRET'),
|
||||
accessSecret: await testQenv.getEnvVarOnDemandStrict('S3_SECRETKEY'),
|
||||
endpoint: await testQenv.getEnvVarOnDemandStrict('S3_ENDPOINT'),
|
||||
port: parseInt(await testQenv.getEnvVarOnDemandStrict('S3_PORT')),
|
||||
useSsl: false,
|
||||
});
|
||||
expect(testSmartbucket).toBeInstanceOf(smartbucket.SmartBucket);
|
||||
myBucket = await testSmartbucket.getBucketByName(await testQenv.getEnvVarOnDemandStrict('S3_BUCKET'),);
|
||||
const bucketName = await testQenv.getEnvVarOnDemandStrict('S3_BUCKET');
|
||||
myBucket = await testSmartbucket.getBucketByName(bucketName);
|
||||
expect(myBucket).toBeInstanceOf(smartbucket.Bucket);
|
||||
expect(myBucket.name).toEqual('test-pushrocks-smartbucket');
|
||||
});
|
||||
|
||||
tap.test('should clean all contents', async () => {
|
||||
|
||||
Reference in New Issue
Block a user