import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as plugins from '../../../ts/plugins.ts';
import { EInvoice } from '../../../ts/classes.xinvoice.ts';
import { CorpusLoader } from '../../helpers/corpus.loader.ts';
import { PerformanceTracker } from '../../helpers/performance.tracker.ts';
const testTimeout = 300000; // 5 minutes timeout for conversion processing
// CONV-06: Data Loss Detection
// Tests detection and reporting of data loss during format conversions
// including field mapping limitations, unsupported features, and precision loss
tap.test('CONV-06: Data Loss Detection - Field Mapping Loss', async (tools) => {
const startTime = Date.now();
// Test data loss detection during conversions with rich data
const richDataUblXml = `
DATA-LOSS-TEST-001
2024-01-15
380
EUR
Rich data invoice for data loss detection testing
2024-01-01
2024-01-31
January 2024 billing period
ORDER-12345
2023-12-15
BILLING-REF-678
DESPATCH-890
RECEIPT-ABC
CONTRACT-XYZ
ADDITIONAL-DOC-123
Specification
UERGIGNvbnRlbnQgRXhhbXBsZQ==
1234567890123
Rich Data Supplier Ltd
Innovation Street 123
Building A, Floor 5
Tech City
12345
Tech State
Additional address information
DE
DE123456789
VAT
Rich Data Supplier Limited
HRB123456
John Doe
+49-30-12345678
+49-30-12345679
john.doe@richdata.com
9876543210987
Rich Data Customer GmbH
Customer Boulevard 456
Customer City
54321
DE
Delivery Street 789
Delivery City
98765
DE
2024-01-10
58
PAYMENT-ID-456
DE89370400440532013000
Rich Data Account
COBADEFFXXX
Payment due within 30 days. 2% discount if paid within 10 days.
false
95
Volume discount
10.00
100.00
0.1
1
2
90.00
ORDER-LINE-1
-
Premium product with rich metadata
Rich Data Product Pro
BUYER-SKU-123
SELLER-SKU-456
MFG-SKU-789
1234567890123
SPEC-DOC-001
DE
43211508
19.00
VAT
Color
Blue
Weight
2.5
2.5
50.00
1
17.10
90.00
17.10
19.00
VAT
100.00
10.00
90.00
107.10
107.10
`;
try {
const invoice = new EInvoice();
const parseResult = await invoice.fromXmlString(richDataUblXml);
expect(parseResult).toBeTruthy();
// Extract original data elements for comparison
const originalData = {
invoicePeriod: richDataUblXml.includes('InvoicePeriod'),
orderReference: richDataUblXml.includes('OrderReference'),
billingReference: richDataUblXml.includes('BillingReference'),
additionalDocuments: richDataUblXml.includes('AdditionalDocumentReference'),
embeddedDocuments: richDataUblXml.includes('EmbeddedDocumentBinaryObject'),
contactInformation: richDataUblXml.includes('Contact'),
deliveryInformation: richDataUblXml.includes('Delivery'),
paymentMeans: richDataUblXml.includes('PaymentMeans'),
allowanceCharges: richDataUblXml.includes('AllowanceCharge'),
itemProperties: richDataUblXml.includes('AdditionalItemProperty'),
itemIdentifications: richDataUblXml.includes('BuyersItemIdentification'),
taxDetails: richDataUblXml.includes('TaxSubtotal')
};
tools.log('Original UBL data elements detected:');
Object.entries(originalData).forEach(([key, value]) => {
tools.log(` ${key}: ${value}`);
});
// Test conversion and data loss detection
const conversionTargets = ['CII', 'XRECHNUNG'];
for (const target of conversionTargets) {
tools.log(`\nTesting data loss in UBL to ${target} conversion...`);
try {
if (typeof invoice.convertTo === 'function') {
const conversionResult = await invoice.convertTo(target);
if (conversionResult) {
const convertedXml = await conversionResult.toXmlString();
// Check for data preservation
const preservedData = {
invoicePeriod: convertedXml.includes('Period') || convertedXml.includes('BillingPeriod'),
orderReference: convertedXml.includes('ORDER-12345') || convertedXml.includes('OrderReference'),
billingReference: convertedXml.includes('BILLING-REF-678') || convertedXml.includes('BillingReference'),
additionalDocuments: convertedXml.includes('ADDITIONAL-DOC-123') || convertedXml.includes('AdditionalDocument'),
embeddedDocuments: convertedXml.includes('UERGIGNvbnRlbnQgRXhhbXBsZQ==') || convertedXml.includes('EmbeddedDocument'),
contactInformation: convertedXml.includes('john.doe@richdata.com') || convertedXml.includes('Contact'),
deliveryInformation: convertedXml.includes('Delivery Street') || convertedXml.includes('Delivery'),
paymentMeans: convertedXml.includes('DE89370400440532013000') || convertedXml.includes('PaymentMeans'),
allowanceCharges: convertedXml.includes('Volume discount') || convertedXml.includes('Allowance'),
itemProperties: convertedXml.includes('Color') || convertedXml.includes('Blue'),
itemIdentifications: convertedXml.includes('BUYER-SKU-123') || convertedXml.includes('ItemIdentification'),
taxDetails: convertedXml.includes('17.10') && convertedXml.includes('19.00')
};
tools.log(`Data preservation in ${target} format:`);
let preservedCount = 0;
let totalElements = 0;
Object.entries(preservedData).forEach(([key, preserved]) => {
const wasOriginal = originalData[key];
tools.log(` ${key}: ${wasOriginal ? (preserved ? 'PRESERVED' : 'LOST') : 'N/A'}`);
if (wasOriginal) {
totalElements++;
if (preserved) preservedCount++;
}
});
const preservationRate = totalElements > 0 ? (preservedCount / totalElements) * 100 : 0;
const dataLossRate = 100 - preservationRate;
tools.log(`\n${target} Conversion Results:`);
tools.log(` Elements preserved: ${preservedCount}/${totalElements}`);
tools.log(` Preservation rate: ${preservationRate.toFixed(1)}%`);
tools.log(` Data loss rate: ${dataLossRate.toFixed(1)}%`);
if (dataLossRate > 0) {
tools.log(` ⚠ Data loss detected in ${target} conversion`);
// Identify specific losses
const lostElements = Object.entries(preservedData)
.filter(([key, preserved]) => originalData[key] && !preserved)
.map(([key]) => key);
if (lostElements.length > 0) {
tools.log(` Lost elements: ${lostElements.join(', ')}`);
}
} else {
tools.log(` ✓ No data loss detected in ${target} conversion`);
}
// Test if data loss detection is available in the API
if (typeof conversionResult.getDataLossReport === 'function') {
try {
const dataLossReport = await conversionResult.getDataLossReport();
if (dataLossReport) {
tools.log(` Data loss report available: ${dataLossReport.lostFields?.length || 0} lost fields`);
}
} catch (reportError) {
tools.log(` Data loss report error: ${reportError.message}`);
}
}
} else {
tools.log(` ⚠ ${target} conversion returned no result`);
}
} else {
tools.log(` ⚠ ${target} conversion not supported`);
}
} catch (conversionError) {
tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`);
}
}
} catch (error) {
tools.log(`Field mapping loss test failed: ${error.message}`);
}
const duration = Date.now() - startTime;
PerformanceTracker.recordMetric('data-loss-field-mapping', duration);
});
tap.test('CONV-06: Data Loss Detection - Precision Loss', async (tools) => {
const startTime = Date.now();
// Test precision loss in numeric values during conversion
const precisionTestXml = `
PRECISION-TEST-001
2024-01-15
380
EUR
1
3.14159
33.33333
-
Precision Test Product
Precise Weight
2.718281828
Very Precise Measurement
1.4142135623730951
10.617
6.33333
33.33333
6.33333
19.00000
33.33333
33.33333
39.66666
39.66666
`;
try {
const invoice = new EInvoice();
const parseResult = await invoice.fromXmlString(precisionTestXml);
if (parseResult) {
tools.log('Testing precision loss during format conversion...');
// Extract original precision values
const originalPrecisionValues = {
quantity: '3.14159',
lineAmount: '33.33333',
priceAmount: '10.617',
taxAmount: '6.33333',
preciseWeight: '2.718281828',
veryPreciseMeasurement: '1.4142135623730951'
};
const conversionTargets = ['CII'];
for (const target of conversionTargets) {
tools.log(`\nTesting precision preservation in ${target} conversion...`);
try {
if (typeof invoice.convertTo === 'function') {
const conversionResult = await invoice.convertTo(target);
if (conversionResult) {
const convertedXml = await conversionResult.toXmlString();
// Check precision preservation
const precisionPreservation = {};
let totalPrecisionTests = 0;
let precisionPreserved = 0;
Object.entries(originalPrecisionValues).forEach(([key, originalValue]) => {
totalPrecisionTests++;
const isPreserved = convertedXml.includes(originalValue);
precisionPreservation[key] = isPreserved;
if (isPreserved) {
precisionPreserved++;
tools.log(` ✓ ${key}: ${originalValue} preserved`);
} else {
// Check for rounded values
const rounded2 = parseFloat(originalValue).toFixed(2);
const rounded3 = parseFloat(originalValue).toFixed(3);
if (convertedXml.includes(rounded2)) {
tools.log(` ⚠ ${key}: ${originalValue} → ${rounded2} (rounded to 2 decimals)`);
} else if (convertedXml.includes(rounded3)) {
tools.log(` ⚠ ${key}: ${originalValue} → ${rounded3} (rounded to 3 decimals)`);
} else {
tools.log(` ✗ ${key}: ${originalValue} lost or heavily modified`);
}
}
});
const precisionRate = totalPrecisionTests > 0 ? (precisionPreserved / totalPrecisionTests) * 100 : 0;
const precisionLossRate = 100 - precisionRate;
tools.log(`\n${target} Precision Results:`);
tools.log(` Values with full precision: ${precisionPreserved}/${totalPrecisionTests}`);
tools.log(` Precision preservation rate: ${precisionRate.toFixed(1)}%`);
tools.log(` Precision loss rate: ${precisionLossRate.toFixed(1)}%`);
if (precisionLossRate > 0) {
tools.log(` ⚠ Precision loss detected - may be due to format limitations`);
} else {
tools.log(` ✓ Full precision maintained`);
}
} else {
tools.log(` ⚠ ${target} conversion returned no result`);
}
} else {
tools.log(` ⚠ ${target} conversion not supported`);
}
} catch (conversionError) {
tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`);
}
}
} else {
tools.log('⚠ Precision test - UBL parsing failed');
}
} catch (error) {
tools.log(`Precision loss test failed: ${error.message}`);
}
const duration = Date.now() - startTime;
PerformanceTracker.recordMetric('data-loss-precision', duration);
});
tap.test('CONV-06: Data Loss Detection - Unsupported Features', async (tools) => {
const startTime = Date.now();
// Test handling of format-specific features that may not be supported in target format
const unsupportedFeaturesTests = [
{
name: 'UBL Specific Features',
xml: `
UNSUPPORTED-UBL-001
2024-01-15
380
550e8400-e29b-41d4-a716-446655440000
urn:fdc:peppol.eu:2017:poacc:billing:01:1.0
urn:fdc:peppol.eu:2017:poacc:billing:01:1.0
Different Customer Structure
Tax Representative
PROJECT-123
`,
features: ['UUID', 'ProfileExecutionID', 'BuyerCustomerParty', 'TaxRepresentativeParty', 'ProjectReference']
},
{
name: 'Advanced Payment Features',
xml: `
PAYMENT-FEATURES-001
2024-01-15
380
50.00
2024-01-01
31
2024-02-15
INSTRUCTION-789
ONLINE
2.00
1.50
PAYMENT-MEANS-ABC
`,
features: ['PrepaidPayment', 'PaymentDueDate', 'InstructionID', 'PaymentChannelCode', 'SettlementDiscountPercent', 'PenaltySurchargePercent']
}
];
for (const featureTest of unsupportedFeaturesTests) {
tools.log(`\nTesting unsupported features: ${featureTest.name}`);
try {
const invoice = new EInvoice();
const parseResult = await invoice.fromXmlString(featureTest.xml);
if (parseResult) {
// Test conversion to different formats
const targets = ['CII'];
for (const target of targets) {
tools.log(` Converting to ${target}...`);
try {
if (typeof invoice.convertTo === 'function') {
const conversionResult = await invoice.convertTo(target);
if (conversionResult) {
const convertedXml = await conversionResult.toXmlString();
// Check for feature preservation
const featurePreservation = {};
let preservedFeatures = 0;
let totalFeatures = featureTest.features.length;
featureTest.features.forEach(feature => {
const isPreserved = convertedXml.includes(feature) ||
convertedXml.toLowerCase().includes(feature.toLowerCase());
featurePreservation[feature] = isPreserved;
if (isPreserved) {
preservedFeatures++;
tools.log(` ✓ ${feature}: preserved`);
} else {
tools.log(` ✗ ${feature}: not preserved (may be unsupported)`);
}
});
const featurePreservationRate = totalFeatures > 0 ? (preservedFeatures / totalFeatures) * 100 : 0;
const featureLossRate = 100 - featurePreservationRate;
tools.log(` ${target} Feature Support:`);
tools.log(` Preserved features: ${preservedFeatures}/${totalFeatures}`);
tools.log(` Feature preservation rate: ${featurePreservationRate.toFixed(1)}%`);
tools.log(` Feature loss rate: ${featureLossRate.toFixed(1)}%`);
if (featureLossRate > 50) {
tools.log(` ⚠ High feature loss - target format may not support these features`);
} else if (featureLossRate > 0) {
tools.log(` ⚠ Some features lost - partial support in target format`);
} else {
tools.log(` ✓ All features preserved`);
}
} else {
tools.log(` ⚠ ${target} conversion returned no result`);
}
} else {
tools.log(` ⚠ ${target} conversion not supported`);
}
} catch (conversionError) {
tools.log(` ✗ ${target} conversion failed: ${conversionError.message}`);
}
}
} else {
tools.log(` ⚠ ${featureTest.name} UBL parsing failed`);
}
} catch (error) {
tools.log(` ✗ ${featureTest.name} test failed: ${error.message}`);
}
}
const duration = Date.now() - startTime;
PerformanceTracker.recordMetric('data-loss-unsupported-features', duration);
});
tap.test('CONV-06: Data Loss Detection - Round-Trip Loss Analysis', async (tools) => {
const startTime = Date.now();
// Test data loss in round-trip conversions (UBL → CII → UBL)
const roundTripTestXml = `
ROUND-TRIP-001
2024-01-15
380
EUR
Round-trip conversion test
Round Trip Supplier
Round Trip Street 123
Round Trip City
12345
DE
1
1.5
75.50
-
Round Trip Product
Product for round-trip testing
50.33
75.50
75.50
89.85
89.85
`;
try {
const originalInvoice = new EInvoice();
const parseResult = await originalInvoice.fromXmlString(roundTripTestXml);
if (parseResult) {
tools.log('Testing round-trip data loss (UBL → CII → UBL)...');
// Extract key data from original
const originalData = {
id: 'ROUND-TRIP-001',
supplierName: 'Round Trip Supplier',
streetName: 'Round Trip Street 123',
cityName: 'Round Trip City',
postalCode: '12345',
productName: 'Round Trip Product',
quantity: '1.5',
price: '50.33',
lineAmount: '75.50',
payableAmount: '89.85'
};
try {
// Step 1: UBL → CII
if (typeof originalInvoice.convertTo === 'function') {
const ciiInvoice = await originalInvoice.convertTo('CII');
if (ciiInvoice) {
tools.log('✓ Step 1: UBL → CII conversion completed');
const ciiXml = await ciiInvoice.toXmlString();
// Check data preservation in CII
const ciiPreservation = {};
let ciiPreserved = 0;
Object.entries(originalData).forEach(([key, value]) => {
const isPreserved = ciiXml.includes(value);
ciiPreservation[key] = isPreserved;
if (isPreserved) ciiPreserved++;
});
const ciiPreservationRate = (ciiPreserved / Object.keys(originalData).length) * 100;
tools.log(` CII preservation rate: ${ciiPreservationRate.toFixed(1)}%`);
// Step 2: CII → UBL (round-trip)
if (typeof ciiInvoice.convertTo === 'function') {
const roundTripInvoice = await ciiInvoice.convertTo('UBL');
if (roundTripInvoice) {
tools.log('✓ Step 2: CII → UBL conversion completed');
const roundTripXml = await roundTripInvoice.toXmlString();
// Check data preservation after round-trip
const roundTripPreservation = {};
let roundTripPreserved = 0;
Object.entries(originalData).forEach(([key, value]) => {
const isPreserved = roundTripXml.includes(value);
roundTripPreservation[key] = isPreserved;
if (isPreserved) roundTripPreserved++;
const originalPresent = originalData[key];
const ciiPresent = ciiPreservation[key];
const roundTripPresent = isPreserved;
let status = 'LOST';
if (roundTripPresent) status = 'PRESERVED';
else if (ciiPresent) status = 'LOST_IN_ROUND_TRIP';
else status = 'LOST_IN_FIRST_CONVERSION';
tools.log(` ${key}: ${status}`);
});
const roundTripPreservationRate = (roundTripPreserved / Object.keys(originalData).length) * 100;
const totalDataLoss = 100 - roundTripPreservationRate;
tools.log(`\nRound-Trip Analysis Results:`);
tools.log(` Original elements: ${Object.keys(originalData).length}`);
tools.log(` After CII conversion: ${ciiPreserved} preserved (${ciiPreservationRate.toFixed(1)}%)`);
tools.log(` After round-trip: ${roundTripPreserved} preserved (${roundTripPreservationRate.toFixed(1)}%)`);
tools.log(` Total data loss: ${totalDataLoss.toFixed(1)}%`);
if (totalDataLoss === 0) {
tools.log(` ✓ Perfect round-trip - no data loss`);
} else if (totalDataLoss < 20) {
tools.log(` ✓ Good round-trip - minimal data loss`);
} else if (totalDataLoss < 50) {
tools.log(` ⚠ Moderate round-trip data loss`);
} else {
tools.log(` ✗ High round-trip data loss`);
}
// Compare file sizes
const originalSize = roundTripTestXml.length;
const roundTripSize = roundTripXml.length;
const sizeDifference = Math.abs(roundTripSize - originalSize);
const sizeChangePercent = (sizeDifference / originalSize) * 100;
tools.log(` Size analysis:`);
tools.log(` Original: ${originalSize} chars`);
tools.log(` Round-trip: ${roundTripSize} chars`);
tools.log(` Size change: ${sizeChangePercent.toFixed(1)}%`);
} else {
tools.log('⚠ Step 2: CII → UBL conversion returned no result');
}
} else {
tools.log('⚠ Step 2: CII → UBL conversion not supported');
}
} else {
tools.log('⚠ Step 1: UBL → CII conversion returned no result');
}
} else {
tools.log('⚠ Round-trip conversion not supported');
}
} catch (conversionError) {
tools.log(`Round-trip conversion failed: ${conversionError.message}`);
}
} else {
tools.log('⚠ Round-trip test - original UBL parsing failed');
}
} catch (error) {
tools.log(`Round-trip loss analysis failed: ${error.message}`);
}
const duration = Date.now() - startTime;
PerformanceTracker.recordMetric('data-loss-round-trip', duration);
});
tap.test('CONV-06: Performance Summary', async (tools) => {
const operations = [
'data-loss-field-mapping',
'data-loss-precision',
'data-loss-unsupported-features',
'data-loss-round-trip'
];
tools.log(`\n=== Data Loss Detection Performance Summary ===`);
for (const operation of operations) {
const summary = await PerformanceTracker.getSummary(operation);
if (summary) {
tools.log(`${operation}:`);
tools.log(` avg=${summary.average}ms, min=${summary.min}ms, max=${summary.max}ms, p95=${summary.p95}ms`);
}
}
tools.log(`\nData loss detection testing completed.`);
tools.log(`Note: Some data loss is expected when converting between different formats`);
tools.log(`due to format-specific features and structural differences.`);
});