/**
* @file test.perf-07.concurrent-processing.ts
* @description Performance tests for concurrent processing capabilities
*/
import { tap } from '@git.zone/tstest/tapbundle';
import * as plugins from '../../plugins.js';
import { EInvoice } from '../../../ts/index.js';
import { FormatDetector } from '../../../ts/formats/utils/format.detector.js';
import { CorpusLoader } from '../../suite/corpus.loader.js';
import { PerformanceTracker } from '../../suite/performance.tracker.js';
import * as os from 'os';
const performanceTracker = new PerformanceTracker('PERF-07: Concurrent Processing');
tap.test('PERF-07: Concurrent Processing - should handle concurrent operations efficiently', async () => {
// Test 1: Concurrent format detection
await performanceTracker.measureAsync(
'concurrent-format-detection',
async () => {
// Create test data with different formats
const testData = [
...Array(25).fill(null).map((_, i) => ({
id: `ubl-${i}`,
content: `UBL-${i}`
})),
...Array(25).fill(null).map((_, i) => ({
id: `cii-${i}`,
content: `CII-${i}`
})),
...Array(25).fill(null).map((_, i) => ({
id: `unknown-${i}`,
content: `UNKNOWN-${i}`
}))
];
// Test different concurrency levels
const levels = [1, 4, 8, 16, 32];
console.log('\nConcurrent Format Detection:');
console.log('Concurrency | Duration | Throughput | Accuracy');
console.log('------------|----------|------------|----------');
for (const concurrency of levels) {
const startTime = Date.now();
let completed = 0;
let correct = 0;
// Process in batches
for (let i = 0; i < testData.length; i += concurrency) {
const batch = testData.slice(i, i + concurrency);
const promises = batch.map(async (item) => {
const format = await FormatDetector.detectFormat(item.content);
completed++;
// Verify correctness
if ((item.id.startsWith('ubl') && format === 'ubl') ||
(item.id.startsWith('cii') && format === 'cii') ||
(item.id.startsWith('unknown') && format === 'unknown')) {
correct++;
}
return format;
});
await Promise.all(promises);
}
const duration = Date.now() - startTime;
const throughput = (completed / (duration / 1000));
const accuracy = ((correct / completed) * 100).toFixed(2);
console.log(`${String(concurrency).padEnd(11)} | ${String(duration + 'ms').padEnd(8)} | ${throughput.toFixed(2).padEnd(10)}/s | ${accuracy}%`);
}
}
);
// Test 2: Concurrent validation
await performanceTracker.measureAsync(
'concurrent-validation',
async () => {
console.log('\nConcurrent Validation:');
// Create test invoice XMLs
const createInvoiceXml = (id: number, itemCount: number) => {
const items = Array.from({ length: itemCount }, (_, i) => `
${i + 1}
1
100.00
Item ${i + 1}
`).join('');
return `
INV-${id}
2024-02-20
Test Seller
Test Buyer
${(itemCount * 100).toFixed(2)}
${(itemCount * 100).toFixed(2)}
${items}
`;
};
// Test scenarios
const scenarios = [
{ name: 'Small invoices (5 items)', count: 30, itemCount: 5 },
{ name: 'Medium invoices (20 items)', count: 20, itemCount: 20 },
{ name: 'Large invoices (50 items)', count: 10, itemCount: 50 }
];
for (const scenario of scenarios) {
console.log(`\n${scenario.name}:`);
const invoices = Array.from({ length: scenario.count }, (_, i) =>
createInvoiceXml(i, scenario.itemCount)
);
const concurrency = 8;
const startTime = Date.now();
let validCount = 0;
// Process concurrently
for (let i = 0; i < invoices.length; i += concurrency) {
const batch = invoices.slice(i, i + concurrency);
const results = await Promise.all(
batch.map(async (invoiceXml) => {
try {
const einvoice = await EInvoice.fromXml(invoiceXml);
const result = await einvoice.validate();
return result.isValid;
} catch {
return false;
}
})
);
validCount += results.filter(v => v).length;
}
const duration = Date.now() - startTime;
const throughput = (scenario.count / (duration / 1000)).toFixed(2);
const validationRate = ((validCount / scenario.count) * 100).toFixed(2);
console.log(` - Processed: ${scenario.count} invoices`);
console.log(` - Duration: ${duration}ms`);
console.log(` - Throughput: ${throughput} invoices/sec`);
console.log(` - Validation rate: ${validationRate}%`);
}
}
);
// Test 3: Concurrent file processing
await performanceTracker.measureAsync(
'concurrent-file-processing',
async () => {
console.log('\nConcurrent File Processing:');
const testDataset = await CorpusLoader.createTestDataset({
formats: ['UBL', 'CII'],
maxFiles: 50,
validOnly: true
});
const files = testDataset.map(f => f.path).filter(p => p.endsWith('.xml'));
console.log(`Processing ${files.length} files from corpus...`);
// Test different concurrency strategies
const strategies = [
{ name: 'Sequential', concurrency: 1 },
{ name: 'Moderate', concurrency: 8 },
{ name: 'Aggressive', concurrency: 16 }
];
for (const strategy of strategies) {
const startTime = Date.now();
let processed = 0;
let errors = 0;
// Process files with specified concurrency
const queue = [...files];
const activePromises = new Set>();
while (queue.length > 0 || activePromises.size > 0) {
// Start new tasks up to concurrency limit
while (activePromises.size < strategy.concurrency && queue.length > 0) {
const file = queue.shift()!;
const promise = (async () => {
try {
const content = await plugins.fs.readFile(file, 'utf-8');
const format = await FormatDetector.detectFormat(content);
if (format && format !== 'unknown' && format !== 'pdf' && format !== 'xml') {
try {
const invoice = await EInvoice.fromXml(content);
await invoice.validate();
processed++;
} catch {
// Skip unparseable files
}
}
} catch {
errors++;
}
})();
activePromises.add(promise);
promise.finally(() => activePromises.delete(promise));
}
// Wait for at least one to complete
if (activePromises.size > 0) {
await Promise.race(activePromises);
}
}
const duration = Date.now() - startTime;
const throughput = (processed / (duration / 1000)).toFixed(2);
console.log(`\n${strategy.name} (concurrency: ${strategy.concurrency}):`);
console.log(` - Duration: ${duration}ms`);
console.log(` - Processed: ${processed} files`);
console.log(` - Throughput: ${throughput} files/sec`);
console.log(` - Errors: ${errors}`);
}
}
);
// Test 4: Mixed operations
await performanceTracker.measureAsync(
'mixed-operations',
async () => {
console.log('\nMixed Operations Concurrency:');
// Define operations
const operations = [
{
name: 'detect',
fn: async () => {
const xml = `TEST`;
return await FormatDetector.detectFormat(xml);
}
},
{
name: 'parse',
fn: async () => {
const xml = `TEST2024-01-01`;
const invoice = await EInvoice.fromXml(xml);
return invoice.getFormat();
}
},
{
name: 'validate',
fn: async () => {
const xml = `
TEST
2024-02-20
Seller
Buyer
`;
const invoice = await EInvoice.fromXml(xml);
return await invoice.validate();
}
}
];
// Test mixed workload
const totalOperations = 150;
const operationMix = Array.from({ length: totalOperations }, (_, i) => ({
operation: operations[i % operations.length],
id: i
}));
const concurrency = 10;
const startTime = Date.now();
const operationCounts = new Map(operations.map(op => [op.name, 0]));
// Process operations
for (let i = 0; i < operationMix.length; i += concurrency) {
const batch = operationMix.slice(i, i + concurrency);
await Promise.all(batch.map(async ({ operation }) => {
try {
await operation.fn();
operationCounts.set(operation.name, operationCounts.get(operation.name)! + 1);
} catch {
// Ignore errors
}
}));
}
const totalDuration = Date.now() - startTime;
const throughput = (totalOperations / (totalDuration / 1000)).toFixed(2);
console.log(` Total operations: ${totalOperations}`);
console.log(` Duration: ${totalDuration}ms`);
console.log(` Throughput: ${throughput} ops/sec`);
console.log(` Operation breakdown:`);
operationCounts.forEach((count, name) => {
console.log(` - ${name}: ${count} operations`);
});
}
);
// Test 5: Resource contention
await performanceTracker.measureAsync(
'resource-contention',
async () => {
console.log('\nResource Contention Test:');
const xml = `
CONTENTION-TEST
2024-02-20
Seller
Buyer
`;
const concurrencyLevels = [1, 10, 50, 100];
console.log('Concurrency | Duration | Throughput');
console.log('------------|----------|------------');
for (const level of concurrencyLevels) {
const start = Date.now();
const promises = Array(level).fill(null).map(async () => {
const invoice = await EInvoice.fromXml(xml);
return invoice.validate();
});
await Promise.all(promises);
const duration = Date.now() - start;
const throughput = (level / (duration / 1000)).toFixed(2);
console.log(`${String(level).padEnd(11)} | ${String(duration + 'ms').padEnd(8)} | ${throughput} ops/sec`);
}
}
);
// Overall summary
console.log('\n=== PERF-07: Overall Performance Summary ===');
console.log(performanceTracker.getSummary());
});
tap.start();