import { tap, expect } from '@git.zone/tstest/tapbundle'; import * as plugins from '../../../ts/plugins.ts'; import { EInvoice } from '../../../ts/classes.xinvoice.ts'; import { CorpusLoader } from '../../helpers/corpus.loader.ts'; import { PerformanceTracker } from '../../helpers/performance.tracker.ts'; const testTimeout = 300000; // 5 minutes timeout for conversion processing // CONV-06: Data Loss Detection // Tests detection and reporting of data loss during format conversions // including field mapping limitations, unsupported features, and precision loss tap.test('CONV-06: Data Loss Detection - Field Mapping Loss', async (tools) => { const startTime = Date.now(); // Test data loss detection during conversions with rich data const richDataUblXml = ` DATA-LOSS-TEST-001 2024-01-15 380 EUR Rich data invoice for data loss detection testing 2024-01-01 2024-01-31 January 2024 billing period ORDER-12345 2023-12-15 BILLING-REF-678 DESPATCH-890 RECEIPT-ABC CONTRACT-XYZ ADDITIONAL-DOC-123 Specification UERGIGNvbnRlbnQgRXhhbXBsZQ== 1234567890123 Rich Data Supplier Ltd Innovation Street 123 Building A, Floor 5 Tech City 12345 Tech State Additional address information DE DE123456789 VAT Rich Data Supplier Limited HRB123456 John Doe +49-30-12345678 +49-30-12345679 john.doe@richdata.com 9876543210987 Rich Data Customer GmbH Customer Boulevard 456 Customer City 54321 DE
Delivery Street 789 Delivery City 98765 DE
2024-01-10
58 PAYMENT-ID-456 DE89370400440532013000 Rich Data Account COBADEFFXXX Payment due within 30 days. 2% discount if paid within 10 days. false 95 Volume discount 10.00 100.00 0.1 1 2 90.00 ORDER-LINE-1 Premium product with rich metadata Rich Data Product Pro BUYER-SKU-123 SELLER-SKU-456 MFG-SKU-789 1234567890123 SPEC-DOC-001 DE 43211508 19.00 VAT Color Blue Weight 2.5 2.5 50.00 1 17.10 90.00 17.10 19.00 VAT 100.00 10.00 90.00 107.10 107.10
`; try { const invoice = new EInvoice(); const parseResult = await invoice.fromXmlString(richDataUblXml); expect(parseResult).toBeTruthy(); // Extract original data elements for comparison const originalData = { invoicePeriod: richDataUblXml.includes('InvoicePeriod'), orderReference: richDataUblXml.includes('OrderReference'), billingReference: richDataUblXml.includes('BillingReference'), additionalDocuments: richDataUblXml.includes('AdditionalDocumentReference'), embeddedDocuments: richDataUblXml.includes('EmbeddedDocumentBinaryObject'), contactInformation: richDataUblXml.includes('Contact'), deliveryInformation: richDataUblXml.includes('Delivery'), paymentMeans: richDataUblXml.includes('PaymentMeans'), allowanceCharges: richDataUblXml.includes('AllowanceCharge'), itemProperties: richDataUblXml.includes('AdditionalItemProperty'), itemIdentifications: richDataUblXml.includes('BuyersItemIdentification'), taxDetails: richDataUblXml.includes('TaxSubtotal') }; tools.log('Original UBL data elements detected:'); Object.entries(originalData).forEach(([key, value]) => { tools.log(` ${key}: ${value}`); }); // Test conversion and data loss detection const conversionTargets = ['CII', 'XRECHNUNG']; for (const target of conversionTargets) { tools.log(`\nTesting data loss in UBL to ${target} conversion...`); try { if (typeof invoice.convertTo === 'function') { const conversionResult = await invoice.convertTo(target); if (conversionResult) { const convertedXml = await conversionResult.toXmlString(); // Check for data preservation const preservedData = { invoicePeriod: convertedXml.includes('Period') || convertedXml.includes('BillingPeriod'), orderReference: convertedXml.includes('ORDER-12345') || convertedXml.includes('OrderReference'), billingReference: convertedXml.includes('BILLING-REF-678') || convertedXml.includes('BillingReference'), additionalDocuments: convertedXml.includes('ADDITIONAL-DOC-123') || convertedXml.includes('AdditionalDocument'), embeddedDocuments: convertedXml.includes('UERGIGNvbnRlbnQgRXhhbXBsZQ==') || convertedXml.includes('EmbeddedDocument'), contactInformation: convertedXml.includes('john.doe@richdata.com') || convertedXml.includes('Contact'), deliveryInformation: convertedXml.includes('Delivery Street') || convertedXml.includes('Delivery'), paymentMeans: convertedXml.includes('DE89370400440532013000') || convertedXml.includes('PaymentMeans'), allowanceCharges: convertedXml.includes('Volume discount') || convertedXml.includes('Allowance'), itemProperties: convertedXml.includes('Color') || convertedXml.includes('Blue'), itemIdentifications: convertedXml.includes('BUYER-SKU-123') || convertedXml.includes('ItemIdentification'), taxDetails: convertedXml.includes('17.10') && convertedXml.includes('19.00') }; tools.log(`Data preservation in ${target} format:`); let preservedCount = 0; let totalElements = 0; Object.entries(preservedData).forEach(([key, preserved]) => { const wasOriginal = originalData[key]; tools.log(` ${key}: ${wasOriginal ? (preserved ? 'PRESERVED' : 'LOST') : 'N/A'}`); if (wasOriginal) { totalElements++; if (preserved) preservedCount++; } }); const preservationRate = totalElements > 0 ? (preservedCount / totalElements) * 100 : 0; const dataLossRate = 100 - preservationRate; tools.log(`\n${target} Conversion Results:`); tools.log(` Elements preserved: ${preservedCount}/${totalElements}`); tools.log(` Preservation rate: ${preservationRate.toFixed(1)}%`); tools.log(` Data loss rate: ${dataLossRate.toFixed(1)}%`); if (dataLossRate > 0) { tools.log(` ⚠ Data loss detected in ${target} conversion`); // Identify specific losses const lostElements = Object.entries(preservedData) .filter(([key, preserved]) => originalData[key] && !preserved) .map(([key]) => key); if (lostElements.length > 0) { tools.log(` Lost elements: ${lostElements.join(', ')}`); } } else { tools.log(` ✓ No data loss detected in ${target} conversion`); } // Test if data loss detection is available in the API if (typeof conversionResult.getDataLossReport === 'function') { try { const dataLossReport = await conversionResult.getDataLossReport(); if (dataLossReport) { tools.log(` Data loss report available: ${dataLossReport.lostFields?.length || 0} lost fields`); } } catch (reportError) { tools.log(` Data loss report error: ${reportError.message}`); } } } else { tools.log(` ⚠ ${target} conversion returned no result`); } } else { tools.log(` ⚠ ${target} conversion not supported`); } } catch (conversionError) { tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`); } } } catch (error) { tools.log(`Field mapping loss test failed: ${error.message}`); } const duration = Date.now() - startTime; PerformanceTracker.recordMetric('data-loss-field-mapping', duration); }); tap.test('CONV-06: Data Loss Detection - Precision Loss', async (tools) => { const startTime = Date.now(); // Test precision loss in numeric values during conversion const precisionTestXml = ` PRECISION-TEST-001 2024-01-15 380 EUR 1 3.14159 33.33333 Precision Test Product Precise Weight 2.718281828 Very Precise Measurement 1.4142135623730951 10.617 6.33333 33.33333 6.33333 19.00000 33.33333 33.33333 39.66666 39.66666 `; try { const invoice = new EInvoice(); const parseResult = await invoice.fromXmlString(precisionTestXml); if (parseResult) { tools.log('Testing precision loss during format conversion...'); // Extract original precision values const originalPrecisionValues = { quantity: '3.14159', lineAmount: '33.33333', priceAmount: '10.617', taxAmount: '6.33333', preciseWeight: '2.718281828', veryPreciseMeasurement: '1.4142135623730951' }; const conversionTargets = ['CII']; for (const target of conversionTargets) { tools.log(`\nTesting precision preservation in ${target} conversion...`); try { if (typeof invoice.convertTo === 'function') { const conversionResult = await invoice.convertTo(target); if (conversionResult) { const convertedXml = await conversionResult.toXmlString(); // Check precision preservation const precisionPreservation = {}; let totalPrecisionTests = 0; let precisionPreserved = 0; Object.entries(originalPrecisionValues).forEach(([key, originalValue]) => { totalPrecisionTests++; const isPreserved = convertedXml.includes(originalValue); precisionPreservation[key] = isPreserved; if (isPreserved) { precisionPreserved++; tools.log(` ✓ ${key}: ${originalValue} preserved`); } else { // Check for rounded values const rounded2 = parseFloat(originalValue).toFixed(2); const rounded3 = parseFloat(originalValue).toFixed(3); if (convertedXml.includes(rounded2)) { tools.log(` ⚠ ${key}: ${originalValue} → ${rounded2} (rounded to 2 decimals)`); } else if (convertedXml.includes(rounded3)) { tools.log(` ⚠ ${key}: ${originalValue} → ${rounded3} (rounded to 3 decimals)`); } else { tools.log(` ✗ ${key}: ${originalValue} lost or heavily modified`); } } }); const precisionRate = totalPrecisionTests > 0 ? (precisionPreserved / totalPrecisionTests) * 100 : 0; const precisionLossRate = 100 - precisionRate; tools.log(`\n${target} Precision Results:`); tools.log(` Values with full precision: ${precisionPreserved}/${totalPrecisionTests}`); tools.log(` Precision preservation rate: ${precisionRate.toFixed(1)}%`); tools.log(` Precision loss rate: ${precisionLossRate.toFixed(1)}%`); if (precisionLossRate > 0) { tools.log(` ⚠ Precision loss detected - may be due to format limitations`); } else { tools.log(` ✓ Full precision maintained`); } } else { tools.log(` ⚠ ${target} conversion returned no result`); } } else { tools.log(` ⚠ ${target} conversion not supported`); } } catch (conversionError) { tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`); } } } else { tools.log('⚠ Precision test - UBL parsing failed'); } } catch (error) { tools.log(`Precision loss test failed: ${error.message}`); } const duration = Date.now() - startTime; PerformanceTracker.recordMetric('data-loss-precision', duration); }); tap.test('CONV-06: Data Loss Detection - Unsupported Features', async (tools) => { const startTime = Date.now(); // Test handling of format-specific features that may not be supported in target format const unsupportedFeaturesTests = [ { name: 'UBL Specific Features', xml: ` UNSUPPORTED-UBL-001 2024-01-15 380 550e8400-e29b-41d4-a716-446655440000 urn:fdc:peppol.eu:2017:poacc:billing:01:1.0 urn:fdc:peppol.eu:2017:poacc:billing:01:1.0 Different Customer Structure Tax Representative PROJECT-123 `, features: ['UUID', 'ProfileExecutionID', 'BuyerCustomerParty', 'TaxRepresentativeParty', 'ProjectReference'] }, { name: 'Advanced Payment Features', xml: ` PAYMENT-FEATURES-001 2024-01-15 380 50.00 2024-01-01 31 2024-02-15 INSTRUCTION-789 ONLINE 2.00 1.50 PAYMENT-MEANS-ABC `, features: ['PrepaidPayment', 'PaymentDueDate', 'InstructionID', 'PaymentChannelCode', 'SettlementDiscountPercent', 'PenaltySurchargePercent'] } ]; for (const featureTest of unsupportedFeaturesTests) { tools.log(`\nTesting unsupported features: ${featureTest.name}`); try { const invoice = new EInvoice(); const parseResult = await invoice.fromXmlString(featureTest.xml); if (parseResult) { // Test conversion to different formats const targets = ['CII']; for (const target of targets) { tools.log(` Converting to ${target}...`); try { if (typeof invoice.convertTo === 'function') { const conversionResult = await invoice.convertTo(target); if (conversionResult) { const convertedXml = await conversionResult.toXmlString(); // Check for feature preservation const featurePreservation = {}; let preservedFeatures = 0; let totalFeatures = featureTest.features.length; featureTest.features.forEach(feature => { const isPreserved = convertedXml.includes(feature) || convertedXml.toLowerCase().includes(feature.toLowerCase()); featurePreservation[feature] = isPreserved; if (isPreserved) { preservedFeatures++; tools.log(` ✓ ${feature}: preserved`); } else { tools.log(` ✗ ${feature}: not preserved (may be unsupported)`); } }); const featurePreservationRate = totalFeatures > 0 ? (preservedFeatures / totalFeatures) * 100 : 0; const featureLossRate = 100 - featurePreservationRate; tools.log(` ${target} Feature Support:`); tools.log(` Preserved features: ${preservedFeatures}/${totalFeatures}`); tools.log(` Feature preservation rate: ${featurePreservationRate.toFixed(1)}%`); tools.log(` Feature loss rate: ${featureLossRate.toFixed(1)}%`); if (featureLossRate > 50) { tools.log(` ⚠ High feature loss - target format may not support these features`); } else if (featureLossRate > 0) { tools.log(` ⚠ Some features lost - partial support in target format`); } else { tools.log(` ✓ All features preserved`); } } else { tools.log(` ⚠ ${target} conversion returned no result`); } } else { tools.log(` ⚠ ${target} conversion not supported`); } } catch (conversionError) { tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`); } } } else { tools.log(` ⚠ ${featureTest.name} UBL parsing failed`); } } catch (error) { tools.log(` ✗ ${featureTest.name} test failed: ${error.message}`); } } const duration = Date.now() - startTime; PerformanceTracker.recordMetric('data-loss-unsupported-features', duration); }); tap.test('CONV-06: Data Loss Detection - Round-Trip Loss Analysis', async (tools) => { const startTime = Date.now(); // Test data loss in round-trip conversions (UBL → CII → UBL) const roundTripTestXml = ` ROUND-TRIP-001 2024-01-15 380 EUR Round-trip conversion test Round Trip Supplier Round Trip Street 123 Round Trip City 12345 DE 1 1.5 75.50 Round Trip Product Product for round-trip testing 50.33 75.50 75.50 89.85 89.85 `; try { const originalInvoice = new EInvoice(); const parseResult = await originalInvoice.fromXmlString(roundTripTestXml); if (parseResult) { tools.log('Testing round-trip data loss (UBL → CII → UBL)...'); // Extract key data from original const originalData = { id: 'ROUND-TRIP-001', supplierName: 'Round Trip Supplier', streetName: 'Round Trip Street 123', cityName: 'Round Trip City', postalCode: '12345', productName: 'Round Trip Product', quantity: '1.5', price: '50.33', lineAmount: '75.50', payableAmount: '89.85' }; try { // Step 1: UBL → CII if (typeof originalInvoice.convertTo === 'function') { const ciiInvoice = await originalInvoice.convertTo('CII'); if (ciiInvoice) { tools.log('✓ Step 1: UBL → CII conversion completed'); const ciiXml = await ciiInvoice.toXmlString(); // Check data preservation in CII const ciiPreservation = {}; let ciiPreserved = 0; Object.entries(originalData).forEach(([key, value]) => { const isPreserved = ciiXml.includes(value); ciiPreservation[key] = isPreserved; if (isPreserved) ciiPreserved++; }); const ciiPreservationRate = (ciiPreserved / Object.keys(originalData).length) * 100; tools.log(` CII preservation rate: ${ciiPreservationRate.toFixed(1)}%`); // Step 2: CII → UBL (round-trip) if (typeof ciiInvoice.convertTo === 'function') { const roundTripInvoice = await ciiInvoice.convertTo('UBL'); if (roundTripInvoice) { tools.log('✓ Step 2: CII → UBL conversion completed'); const roundTripXml = await roundTripInvoice.toXmlString(); // Check data preservation after round-trip const roundTripPreservation = {}; let roundTripPreserved = 0; Object.entries(originalData).forEach(([key, value]) => { const isPreserved = roundTripXml.includes(value); roundTripPreservation[key] = isPreserved; if (isPreserved) roundTripPreserved++; const originalPresent = originalData[key]; const ciiPresent = ciiPreservation[key]; const roundTripPresent = isPreserved; let status = 'LOST'; if (roundTripPresent) status = 'PRESERVED'; else if (ciiPresent) status = 'LOST_IN_ROUND_TRIP'; else status = 'LOST_IN_FIRST_CONVERSION'; tools.log(` ${key}: ${status}`); }); const roundTripPreservationRate = (roundTripPreserved / Object.keys(originalData).length) * 100; const totalDataLoss = 100 - roundTripPreservationRate; tools.log(`\nRound-Trip Analysis Results:`); tools.log(` Original elements: ${Object.keys(originalData).length}`); tools.log(` After CII conversion: ${ciiPreserved} preserved (${ciiPreservationRate.toFixed(1)}%)`); tools.log(` After round-trip: ${roundTripPreserved} preserved (${roundTripPreservationRate.toFixed(1)}%)`); tools.log(` Total data loss: ${totalDataLoss.toFixed(1)}%`); if (totalDataLoss === 0) { tools.log(` ✓ Perfect round-trip - no data loss`); } else if (totalDataLoss < 20) { tools.log(` ✓ Good round-trip - minimal data loss`); } else if (totalDataLoss < 50) { tools.log(` ⚠ Moderate round-trip data loss`); } else { tools.log(` ✗ High round-trip data loss`); } // Compare file sizes const originalSize = roundTripTestXml.length; const roundTripSize = roundTripXml.length; const sizeDifference = Math.abs(roundTripSize - originalSize); const sizeChangePercent = (sizeDifference / originalSize) * 100; tools.log(` Size analysis:`); tools.log(` Original: ${originalSize} chars`); tools.log(` Round-trip: ${roundTripSize} chars`); tools.log(` Size change: ${sizeChangePercent.toFixed(1)}%`); } else { tools.log('⚠ Step 2: CII → UBL conversion returned no result'); } } else { tools.log('⚠ Step 2: CII → UBL conversion not supported'); } } else { tools.log('⚠ Step 1: UBL → CII conversion returned no result'); } } else { tools.log('⚠ Round-trip conversion not supported'); } } catch (conversionError) { tools.log(`Round-trip conversion failed: ${conversionError.message}`); } } else { tools.log('⚠ Round-trip test - original UBL parsing failed'); } } catch (error) { tools.log(`Round-trip loss analysis failed: ${error.message}`); } const duration = Date.now() - startTime; PerformanceTracker.recordMetric('data-loss-round-trip', duration); }); tap.test('CONV-06: Performance Summary', async (tools) => { const operations = [ 'data-loss-field-mapping', 'data-loss-precision', 'data-loss-unsupported-features', 'data-loss-round-trip' ]; tools.log(`\n=== Data Loss Detection Performance Summary ===`); for (const operation of operations) { const summary = await PerformanceTracker.getSummary(operation); if (summary) { tools.log(`${operation}:`); tools.log(` avg=${summary.average}ms, min=${summary.min}ms, max=${summary.max}ms, p95=${summary.p95}ms`); } } tools.log(`\nData loss detection testing completed.`); tools.log(`Note: Some data loss is expected when converting between different formats`); tools.log(`due to format-specific features and structural differences.`); });