update
This commit is contained in:
473
test/suite/einvoice_conversion/test.conv-10.batch-conversion.ts
Normal file
473
test/suite/einvoice_conversion/test.conv-10.batch-conversion.ts
Normal file
@@ -0,0 +1,473 @@
|
||||
/**
|
||||
* @file test.conv-10.batch-conversion.ts
|
||||
* @description Tests for batch conversion operations and performance
|
||||
*/
|
||||
|
||||
import { tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from '../../plugins.js';
|
||||
import { EInvoice } from '../../../ts/index.js';
|
||||
import { CorpusLoader } from '../../suite/corpus.loader.js';
|
||||
import { PerformanceTracker } from '../../suite/performance.tracker.js';
|
||||
|
||||
const corpusLoader = new CorpusLoader();
|
||||
const performanceTracker = new PerformanceTracker('CONV-10: Batch Conversion');
|
||||
|
||||
tap.test('CONV-10: Batch Conversion - should efficiently handle batch conversion operations', async (t) => {
|
||||
// Test 1: Sequential batch conversion
|
||||
const sequentialBatch = await performanceTracker.measureAsync(
|
||||
'sequential-batch-conversion',
|
||||
async () => {
|
||||
const einvoice = new EInvoice();
|
||||
const batchSize = 10;
|
||||
const results = {
|
||||
processed: 0,
|
||||
successful: 0,
|
||||
failed: 0,
|
||||
totalTime: 0,
|
||||
averageTime: 0
|
||||
};
|
||||
|
||||
// Create test invoices
|
||||
const invoices = Array.from({ length: batchSize }, (_, i) => ({
|
||||
format: 'ubl' as const,
|
||||
data: {
|
||||
documentType: 'INVOICE',
|
||||
invoiceNumber: `BATCH-SEQ-2024-${String(i + 1).padStart(3, '0')}`,
|
||||
issueDate: '2024-01-25',
|
||||
seller: {
|
||||
name: `Seller Company ${i + 1}`,
|
||||
address: `Address ${i + 1}`,
|
||||
country: 'DE',
|
||||
taxId: `DE${String(123456789 + i).padStart(9, '0')}`
|
||||
},
|
||||
buyer: {
|
||||
name: `Buyer Company ${i + 1}`,
|
||||
address: `Buyer Address ${i + 1}`,
|
||||
country: 'DE',
|
||||
taxId: `DE${String(987654321 - i).padStart(9, '0')}`
|
||||
},
|
||||
items: [{
|
||||
description: `Product ${i + 1}`,
|
||||
quantity: i + 1,
|
||||
unitPrice: 100.00 + (i * 10),
|
||||
vatRate: 19,
|
||||
lineTotal: (i + 1) * (100.00 + (i * 10))
|
||||
}],
|
||||
totals: {
|
||||
netAmount: (i + 1) * (100.00 + (i * 10)),
|
||||
vatAmount: (i + 1) * (100.00 + (i * 10)) * 0.19,
|
||||
grossAmount: (i + 1) * (100.00 + (i * 10)) * 1.19
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// Process sequentially
|
||||
const startTime = Date.now();
|
||||
|
||||
for (const invoice of invoices) {
|
||||
results.processed++;
|
||||
try {
|
||||
const converted = await einvoice.convertFormat(invoice, 'cii');
|
||||
if (converted) {
|
||||
results.successful++;
|
||||
}
|
||||
} catch (error) {
|
||||
results.failed++;
|
||||
}
|
||||
}
|
||||
|
||||
results.totalTime = Date.now() - startTime;
|
||||
results.averageTime = results.totalTime / results.processed;
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
// Test 2: Parallel batch conversion
|
||||
const parallelBatch = await performanceTracker.measureAsync(
|
||||
'parallel-batch-conversion',
|
||||
async () => {
|
||||
const einvoice = new EInvoice();
|
||||
const batchSize = 10;
|
||||
const results = {
|
||||
processed: 0,
|
||||
successful: 0,
|
||||
failed: 0,
|
||||
totalTime: 0,
|
||||
averageTime: 0
|
||||
};
|
||||
|
||||
// Create test invoices
|
||||
const invoices = Array.from({ length: batchSize }, (_, i) => ({
|
||||
format: 'cii' as const,
|
||||
data: {
|
||||
documentType: 'INVOICE',
|
||||
invoiceNumber: `BATCH-PAR-2024-${String(i + 1).padStart(3, '0')}`,
|
||||
issueDate: '2024-01-25',
|
||||
seller: {
|
||||
name: `Parallel Seller ${i + 1}`,
|
||||
address: `Parallel Address ${i + 1}`,
|
||||
country: 'FR',
|
||||
taxId: `FR${String(12345678901 + i).padStart(11, '0')}`
|
||||
},
|
||||
buyer: {
|
||||
name: `Parallel Buyer ${i + 1}`,
|
||||
address: `Parallel Buyer Address ${i + 1}`,
|
||||
country: 'FR',
|
||||
taxId: `FR${String(98765432109 - i).padStart(11, '0')}`
|
||||
},
|
||||
items: [{
|
||||
description: `Service ${i + 1}`,
|
||||
quantity: 1,
|
||||
unitPrice: 500.00 + (i * 50),
|
||||
vatRate: 20,
|
||||
lineTotal: 500.00 + (i * 50)
|
||||
}],
|
||||
totals: {
|
||||
netAmount: 500.00 + (i * 50),
|
||||
vatAmount: (500.00 + (i * 50)) * 0.20,
|
||||
grossAmount: (500.00 + (i * 50)) * 1.20
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// Process in parallel
|
||||
const startTime = Date.now();
|
||||
|
||||
const conversionPromises = invoices.map(async (invoice) => {
|
||||
try {
|
||||
const converted = await einvoice.convertFormat(invoice, 'ubl');
|
||||
return { success: true, converted };
|
||||
} catch (error) {
|
||||
return { success: false, error };
|
||||
}
|
||||
});
|
||||
|
||||
const conversionResults = await Promise.all(conversionPromises);
|
||||
|
||||
results.processed = conversionResults.length;
|
||||
results.successful = conversionResults.filter(r => r.success).length;
|
||||
results.failed = conversionResults.filter(r => !r.success).length;
|
||||
results.totalTime = Date.now() - startTime;
|
||||
results.averageTime = results.totalTime / results.processed;
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
// Test 3: Mixed format batch conversion
|
||||
const mixedFormatBatch = await performanceTracker.measureAsync(
|
||||
'mixed-format-batch-conversion',
|
||||
async () => {
|
||||
const einvoice = new EInvoice();
|
||||
const formats = ['ubl', 'cii', 'zugferd', 'xrechnung'] as const;
|
||||
const results = {
|
||||
byFormat: new Map<string, { processed: number; successful: number; failed: number }>(),
|
||||
totalProcessed: 0,
|
||||
totalSuccessful: 0,
|
||||
conversionMatrix: new Map<string, number>()
|
||||
};
|
||||
|
||||
// Create mixed format invoices
|
||||
const mixedInvoices = formats.flatMap((format, formatIndex) =>
|
||||
Array.from({ length: 3 }, (_, i) => ({
|
||||
format,
|
||||
data: {
|
||||
documentType: 'INVOICE',
|
||||
invoiceNumber: `MIXED-${format.toUpperCase()}-${i + 1}`,
|
||||
issueDate: '2024-01-26',
|
||||
seller: {
|
||||
name: `${format.toUpperCase()} Seller ${i + 1}`,
|
||||
address: 'Mixed Street 1',
|
||||
country: 'DE',
|
||||
taxId: `DE${String(111111111 + formatIndex * 10 + i).padStart(9, '0')}`
|
||||
},
|
||||
buyer: {
|
||||
name: `${format.toUpperCase()} Buyer ${i + 1}`,
|
||||
address: 'Mixed Avenue 2',
|
||||
country: 'DE',
|
||||
taxId: `DE${String(999999999 - formatIndex * 10 - i).padStart(9, '0')}`
|
||||
},
|
||||
items: [{
|
||||
description: `${format} Product`,
|
||||
quantity: 1,
|
||||
unitPrice: 250.00,
|
||||
vatRate: 19,
|
||||
lineTotal: 250.00
|
||||
}],
|
||||
totals: {
|
||||
netAmount: 250.00,
|
||||
vatAmount: 47.50,
|
||||
grossAmount: 297.50
|
||||
}
|
||||
}
|
||||
}))
|
||||
);
|
||||
|
||||
// Process with different target formats
|
||||
const targetFormats = ['ubl', 'cii'] as const;
|
||||
|
||||
for (const invoice of mixedInvoices) {
|
||||
const sourceFormat = invoice.format;
|
||||
|
||||
if (!results.byFormat.has(sourceFormat)) {
|
||||
results.byFormat.set(sourceFormat, { processed: 0, successful: 0, failed: 0 });
|
||||
}
|
||||
|
||||
const formatStats = results.byFormat.get(sourceFormat)!;
|
||||
|
||||
for (const targetFormat of targetFormats) {
|
||||
if (sourceFormat === targetFormat) continue;
|
||||
|
||||
const conversionKey = `${sourceFormat}->${targetFormat}`;
|
||||
formatStats.processed++;
|
||||
results.totalProcessed++;
|
||||
|
||||
try {
|
||||
const converted = await einvoice.convertFormat(invoice, targetFormat);
|
||||
if (converted) {
|
||||
formatStats.successful++;
|
||||
results.totalSuccessful++;
|
||||
results.conversionMatrix.set(conversionKey,
|
||||
(results.conversionMatrix.get(conversionKey) || 0) + 1
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
formatStats.failed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
formatStats: Array.from(results.byFormat.entries()),
|
||||
totalProcessed: results.totalProcessed,
|
||||
totalSuccessful: results.totalSuccessful,
|
||||
conversionMatrix: Array.from(results.conversionMatrix.entries()),
|
||||
successRate: (results.totalSuccessful / results.totalProcessed * 100).toFixed(2) + '%'
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// Test 4: Large batch with memory monitoring
|
||||
const largeBatchMemory = await performanceTracker.measureAsync(
|
||||
'large-batch-memory-monitoring',
|
||||
async () => {
|
||||
const einvoice = new EInvoice();
|
||||
const batchSize = 50;
|
||||
const memorySnapshots = [];
|
||||
|
||||
// Capture initial memory
|
||||
if (global.gc) global.gc();
|
||||
const initialMemory = process.memoryUsage();
|
||||
|
||||
// Create large batch
|
||||
const largeBatch = Array.from({ length: batchSize }, (_, i) => ({
|
||||
format: 'ubl' as const,
|
||||
data: {
|
||||
documentType: 'INVOICE',
|
||||
invoiceNumber: `LARGE-BATCH-${String(i + 1).padStart(4, '0')}`,
|
||||
issueDate: '2024-01-27',
|
||||
seller: {
|
||||
name: `Large Batch Seller ${i + 1}`,
|
||||
address: `Street ${i + 1}, Building ${i % 10 + 1}`,
|
||||
city: 'Berlin',
|
||||
postalCode: `${10000 + i}`,
|
||||
country: 'DE',
|
||||
taxId: `DE${String(100000000 + i).padStart(9, '0')}`
|
||||
},
|
||||
buyer: {
|
||||
name: `Large Batch Buyer ${i + 1}`,
|
||||
address: `Avenue ${i + 1}, Suite ${i % 20 + 1}`,
|
||||
city: 'Munich',
|
||||
postalCode: `${80000 + i}`,
|
||||
country: 'DE',
|
||||
taxId: `DE${String(200000000 + i).padStart(9, '0')}`
|
||||
},
|
||||
items: Array.from({ length: 5 }, (_, j) => ({
|
||||
description: `Product ${i + 1}-${j + 1} with detailed description`,
|
||||
quantity: j + 1,
|
||||
unitPrice: 50.00 + j * 10,
|
||||
vatRate: 19,
|
||||
lineTotal: (j + 1) * (50.00 + j * 10)
|
||||
})),
|
||||
totals: {
|
||||
netAmount: Array.from({ length: 5 }, (_, j) => (j + 1) * (50.00 + j * 10)).reduce((a, b) => a + b, 0),
|
||||
vatAmount: Array.from({ length: 5 }, (_, j) => (j + 1) * (50.00 + j * 10)).reduce((a, b) => a + b, 0) * 0.19,
|
||||
grossAmount: Array.from({ length: 5 }, (_, j) => (j + 1) * (50.00 + j * 10)).reduce((a, b) => a + b, 0) * 1.19
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// Process in chunks and monitor memory
|
||||
const chunkSize = 10;
|
||||
let processed = 0;
|
||||
let successful = 0;
|
||||
|
||||
for (let i = 0; i < largeBatch.length; i += chunkSize) {
|
||||
const chunk = largeBatch.slice(i, i + chunkSize);
|
||||
|
||||
// Process chunk
|
||||
const chunkResults = await Promise.all(
|
||||
chunk.map(async (invoice) => {
|
||||
try {
|
||||
await einvoice.convertFormat(invoice, 'cii');
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
processed += chunk.length;
|
||||
successful += chunkResults.filter(r => r).length;
|
||||
|
||||
// Capture memory snapshot
|
||||
const currentMemory = process.memoryUsage();
|
||||
memorySnapshots.push({
|
||||
processed,
|
||||
heapUsed: Math.round((currentMemory.heapUsed - initialMemory.heapUsed) / 1024 / 1024 * 100) / 100,
|
||||
external: Math.round((currentMemory.external - initialMemory.external) / 1024 / 1024 * 100) / 100
|
||||
});
|
||||
}
|
||||
|
||||
// Force garbage collection if available
|
||||
if (global.gc) global.gc();
|
||||
const finalMemory = process.memoryUsage();
|
||||
|
||||
return {
|
||||
processed,
|
||||
successful,
|
||||
successRate: (successful / processed * 100).toFixed(2) + '%',
|
||||
memoryIncrease: {
|
||||
heapUsed: Math.round((finalMemory.heapUsed - initialMemory.heapUsed) / 1024 / 1024 * 100) / 100,
|
||||
external: Math.round((finalMemory.external - initialMemory.external) / 1024 / 1024 * 100) / 100
|
||||
},
|
||||
memorySnapshots,
|
||||
averageMemoryPerInvoice: Math.round((finalMemory.heapUsed - initialMemory.heapUsed) / processed / 1024 * 100) / 100
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// Test 5: Corpus batch conversion
|
||||
const corpusBatch = await performanceTracker.measureAsync(
|
||||
'corpus-batch-conversion',
|
||||
async () => {
|
||||
const files = await corpusLoader.getFilesByPattern('**/*.xml');
|
||||
const einvoice = new EInvoice();
|
||||
const batchStats = {
|
||||
totalFiles: 0,
|
||||
processed: 0,
|
||||
converted: 0,
|
||||
failedParsing: 0,
|
||||
failedConversion: 0,
|
||||
formatDistribution: new Map<string, number>(),
|
||||
processingTimes: [] as number[],
|
||||
formats: new Set<string>()
|
||||
};
|
||||
|
||||
// Process a batch of corpus files
|
||||
const batchFiles = files.slice(0, 25);
|
||||
batchStats.totalFiles = batchFiles.length;
|
||||
|
||||
// Process files in parallel batches
|
||||
const batchSize = 5;
|
||||
for (let i = 0; i < batchFiles.length; i += batchSize) {
|
||||
const batch = batchFiles.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(batch.map(async (file) => {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const content = await plugins.fs.readFile(file, 'utf-8');
|
||||
|
||||
// Detect format
|
||||
const format = await einvoice.detectFormat(content);
|
||||
if (!format || format === 'unknown') {
|
||||
batchStats.failedParsing++;
|
||||
return;
|
||||
}
|
||||
|
||||
batchStats.formats.add(format);
|
||||
batchStats.formatDistribution.set(format,
|
||||
(batchStats.formatDistribution.get(format) || 0) + 1
|
||||
);
|
||||
|
||||
// Parse invoice
|
||||
const invoice = await einvoice.parseInvoice(content, format);
|
||||
batchStats.processed++;
|
||||
|
||||
// Try conversion to different format
|
||||
const targetFormat = format === 'ubl' ? 'cii' : 'ubl';
|
||||
try {
|
||||
await einvoice.convertFormat(invoice, targetFormat);
|
||||
batchStats.converted++;
|
||||
} catch (convError) {
|
||||
batchStats.failedConversion++;
|
||||
}
|
||||
|
||||
batchStats.processingTimes.push(Date.now() - startTime);
|
||||
|
||||
} catch (error) {
|
||||
batchStats.failedParsing++;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
// Calculate statistics
|
||||
const avgProcessingTime = batchStats.processingTimes.length > 0 ?
|
||||
batchStats.processingTimes.reduce((a, b) => a + b, 0) / batchStats.processingTimes.length : 0;
|
||||
|
||||
return {
|
||||
...batchStats,
|
||||
formatDistribution: Array.from(batchStats.formatDistribution.entries()),
|
||||
formats: Array.from(batchStats.formats),
|
||||
averageProcessingTime: Math.round(avgProcessingTime),
|
||||
conversionSuccessRate: batchStats.processed > 0 ?
|
||||
(batchStats.converted / batchStats.processed * 100).toFixed(2) + '%' : 'N/A'
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
// Summary
|
||||
t.comment('\n=== CONV-10: Batch Conversion Test Summary ===');
|
||||
t.comment(`\nSequential Batch (${sequentialBatch.result.processed} invoices):`);
|
||||
t.comment(` - Successful: ${sequentialBatch.result.successful}`);
|
||||
t.comment(` - Failed: ${sequentialBatch.result.failed}`);
|
||||
t.comment(` - Total time: ${sequentialBatch.result.totalTime}ms`);
|
||||
t.comment(` - Average time per invoice: ${sequentialBatch.result.averageTime.toFixed(2)}ms`);
|
||||
|
||||
t.comment(`\nParallel Batch (${parallelBatch.result.processed} invoices):`);
|
||||
t.comment(` - Successful: ${parallelBatch.result.successful}`);
|
||||
t.comment(` - Failed: ${parallelBatch.result.failed}`);
|
||||
t.comment(` - Total time: ${parallelBatch.result.totalTime}ms`);
|
||||
t.comment(` - Average time per invoice: ${parallelBatch.result.averageTime.toFixed(2)}ms`);
|
||||
t.comment(` - Speedup vs sequential: ${(sequentialBatch.result.totalTime / parallelBatch.result.totalTime).toFixed(2)}x`);
|
||||
|
||||
t.comment(`\nMixed Format Batch:`);
|
||||
t.comment(` - Total conversions: ${mixedFormatBatch.result.totalProcessed}`);
|
||||
t.comment(` - Success rate: ${mixedFormatBatch.result.successRate}`);
|
||||
t.comment(` - Format statistics:`);
|
||||
mixedFormatBatch.result.formatStats.forEach(([format, stats]) => {
|
||||
t.comment(` * ${format}: ${stats.successful}/${stats.processed} successful`);
|
||||
});
|
||||
|
||||
t.comment(`\nLarge Batch Memory Analysis (${largeBatchMemory.result.processed} invoices):`);
|
||||
t.comment(` - Success rate: ${largeBatchMemory.result.successRate}`);
|
||||
t.comment(` - Memory increase: ${largeBatchMemory.result.memoryIncrease.heapUsed}MB heap`);
|
||||
t.comment(` - Average memory per invoice: ${largeBatchMemory.result.averageMemoryPerInvoice}KB`);
|
||||
|
||||
t.comment(`\nCorpus Batch Conversion (${corpusBatch.result.totalFiles} files):`);
|
||||
t.comment(` - Successfully parsed: ${corpusBatch.result.processed}`);
|
||||
t.comment(` - Successfully converted: ${corpusBatch.result.converted}`);
|
||||
t.comment(` - Conversion success rate: ${corpusBatch.result.conversionSuccessRate}`);
|
||||
t.comment(` - Average processing time: ${corpusBatch.result.averageProcessingTime}ms`);
|
||||
t.comment(` - Formats found: ${corpusBatch.result.formats.join(', ')}`);
|
||||
|
||||
// Performance summary
|
||||
t.comment('\n=== Performance Summary ===');
|
||||
performanceTracker.logSummary();
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
tap.start();
|
||||
Reference in New Issue
Block a user