/** * @file test.perf-07.concurrent-processing.ts * @description Performance tests for concurrent processing capabilities */ import { tap } from '@git.zone/tstest/tapbundle'; import * as plugins from '../../plugins.js'; import { EInvoice } from '../../../ts/index.js'; import { FormatDetector } from '../../../ts/formats/utils/format.detector.js'; import { CorpusLoader } from '../../suite/corpus.loader.js'; import { PerformanceTracker } from '../../suite/performance.tracker.js'; import * as os from 'os'; const performanceTracker = new PerformanceTracker('PERF-07: Concurrent Processing'); tap.test('PERF-07: Concurrent Processing - should handle concurrent operations efficiently', async () => { // Test 1: Concurrent format detection await performanceTracker.measureAsync( 'concurrent-format-detection', async () => { // Create test data with different formats const testData = [ ...Array(25).fill(null).map((_, i) => ({ id: `ubl-${i}`, content: `UBL-${i}` })), ...Array(25).fill(null).map((_, i) => ({ id: `cii-${i}`, content: `CII-${i}` })), ...Array(25).fill(null).map((_, i) => ({ id: `unknown-${i}`, content: `UNKNOWN-${i}` })) ]; // Test different concurrency levels const levels = [1, 4, 8, 16, 32]; console.log('\nConcurrent Format Detection:'); console.log('Concurrency | Duration | Throughput | Accuracy'); console.log('------------|----------|------------|----------'); for (const concurrency of levels) { const startTime = Date.now(); let completed = 0; let correct = 0; // Process in batches for (let i = 0; i < testData.length; i += concurrency) { const batch = testData.slice(i, i + concurrency); const promises = batch.map(async (item) => { const format = await FormatDetector.detectFormat(item.content); completed++; // Verify correctness if ((item.id.startsWith('ubl') && format === 'ubl') || (item.id.startsWith('cii') && format === 'cii') || (item.id.startsWith('unknown') && format === 'unknown')) { correct++; } return format; }); await Promise.all(promises); } const duration = Date.now() - startTime; const throughput = (completed / (duration / 1000)); const accuracy = ((correct / completed) * 100).toFixed(2); console.log(`${String(concurrency).padEnd(11)} | ${String(duration + 'ms').padEnd(8)} | ${throughput.toFixed(2).padEnd(10)}/s | ${accuracy}%`); } } ); // Test 2: Concurrent validation await performanceTracker.measureAsync( 'concurrent-validation', async () => { console.log('\nConcurrent Validation:'); // Create test invoice XMLs const createInvoiceXml = (id: number, itemCount: number) => { const items = Array.from({ length: itemCount }, (_, i) => ` ${i + 1} 1 100.00 Item ${i + 1} `).join(''); return ` INV-${id} 2024-02-20 Test Seller Test Buyer ${(itemCount * 100).toFixed(2)} ${(itemCount * 100).toFixed(2)} ${items} `; }; // Test scenarios const scenarios = [ { name: 'Small invoices (5 items)', count: 30, itemCount: 5 }, { name: 'Medium invoices (20 items)', count: 20, itemCount: 20 }, { name: 'Large invoices (50 items)', count: 10, itemCount: 50 } ]; for (const scenario of scenarios) { console.log(`\n${scenario.name}:`); const invoices = Array.from({ length: scenario.count }, (_, i) => createInvoiceXml(i, scenario.itemCount) ); const concurrency = 8; const startTime = Date.now(); let validCount = 0; // Process concurrently for (let i = 0; i < invoices.length; i += concurrency) { const batch = invoices.slice(i, i + concurrency); const results = await Promise.all( batch.map(async (invoiceXml) => { try { const einvoice = await EInvoice.fromXml(invoiceXml); const result = await einvoice.validate(); return result.isValid; } catch { return false; } }) ); validCount += results.filter(v => v).length; } const duration = Date.now() - startTime; const throughput = (scenario.count / (duration / 1000)).toFixed(2); const validationRate = ((validCount / scenario.count) * 100).toFixed(2); console.log(` - Processed: ${scenario.count} invoices`); console.log(` - Duration: ${duration}ms`); console.log(` - Throughput: ${throughput} invoices/sec`); console.log(` - Validation rate: ${validationRate}%`); } } ); // Test 3: Concurrent file processing await performanceTracker.measureAsync( 'concurrent-file-processing', async () => { console.log('\nConcurrent File Processing:'); const testDataset = await CorpusLoader.createTestDataset({ formats: ['UBL', 'CII'], maxFiles: 50, validOnly: true }); const files = testDataset.map(f => f.path).filter(p => p.endsWith('.xml')); console.log(`Processing ${files.length} files from corpus...`); // Test different concurrency strategies const strategies = [ { name: 'Sequential', concurrency: 1 }, { name: 'Moderate', concurrency: 8 }, { name: 'Aggressive', concurrency: 16 } ]; for (const strategy of strategies) { const startTime = Date.now(); let processed = 0; let errors = 0; // Process files with specified concurrency const queue = [...files]; const activePromises = new Set>(); while (queue.length > 0 || activePromises.size > 0) { // Start new tasks up to concurrency limit while (activePromises.size < strategy.concurrency && queue.length > 0) { const file = queue.shift()!; const promise = (async () => { try { const content = await plugins.fs.readFile(file, 'utf-8'); const format = await FormatDetector.detectFormat(content); if (format && format !== 'unknown' && format !== 'pdf' && format !== 'xml') { try { const invoice = await EInvoice.fromXml(content); await invoice.validate(); processed++; } catch { // Skip unparseable files } } } catch { errors++; } })(); activePromises.add(promise); promise.finally(() => activePromises.delete(promise)); } // Wait for at least one to complete if (activePromises.size > 0) { await Promise.race(activePromises); } } const duration = Date.now() - startTime; const throughput = (processed / (duration / 1000)).toFixed(2); console.log(`\n${strategy.name} (concurrency: ${strategy.concurrency}):`); console.log(` - Duration: ${duration}ms`); console.log(` - Processed: ${processed} files`); console.log(` - Throughput: ${throughput} files/sec`); console.log(` - Errors: ${errors}`); } } ); // Test 4: Mixed operations await performanceTracker.measureAsync( 'mixed-operations', async () => { console.log('\nMixed Operations Concurrency:'); // Define operations const operations = [ { name: 'detect', fn: async () => { const xml = `TEST`; return await FormatDetector.detectFormat(xml); } }, { name: 'parse', fn: async () => { const xml = `TEST2024-01-01`; const invoice = await EInvoice.fromXml(xml); return invoice.getFormat(); } }, { name: 'validate', fn: async () => { const xml = ` TEST 2024-02-20 Seller Buyer `; const invoice = await EInvoice.fromXml(xml); return await invoice.validate(); } } ]; // Test mixed workload const totalOperations = 150; const operationMix = Array.from({ length: totalOperations }, (_, i) => ({ operation: operations[i % operations.length], id: i })); const concurrency = 10; const startTime = Date.now(); const operationCounts = new Map(operations.map(op => [op.name, 0])); // Process operations for (let i = 0; i < operationMix.length; i += concurrency) { const batch = operationMix.slice(i, i + concurrency); await Promise.all(batch.map(async ({ operation }) => { try { await operation.fn(); operationCounts.set(operation.name, operationCounts.get(operation.name)! + 1); } catch { // Ignore errors } })); } const totalDuration = Date.now() - startTime; const throughput = (totalOperations / (totalDuration / 1000)).toFixed(2); console.log(` Total operations: ${totalOperations}`); console.log(` Duration: ${totalDuration}ms`); console.log(` Throughput: ${throughput} ops/sec`); console.log(` Operation breakdown:`); operationCounts.forEach((count, name) => { console.log(` - ${name}: ${count} operations`); }); } ); // Test 5: Resource contention await performanceTracker.measureAsync( 'resource-contention', async () => { console.log('\nResource Contention Test:'); const xml = ` CONTENTION-TEST 2024-02-20 Seller Buyer `; const concurrencyLevels = [1, 10, 50, 100]; console.log('Concurrency | Duration | Throughput'); console.log('------------|----------|------------'); for (const level of concurrencyLevels) { const start = Date.now(); const promises = Array(level).fill(null).map(async () => { const invoice = await EInvoice.fromXml(xml); return invoice.validate(); }); await Promise.all(promises); const duration = Date.now() - start; const throughput = (level / (duration / 1000)).toFixed(2); console.log(`${String(level).padEnd(11)} | ${String(duration + 'ms').padEnd(8)} | ${throughput} ops/sec`); } } ); // Overall summary console.log('\n=== PERF-07: Overall Performance Summary ==='); console.log(performanceTracker.getSummary()); }); tap.start();