fix(compliance): improve compliance
This commit is contained in:
@ -1,674 +1,314 @@
|
||||
import { tap } from '@git.zone/tstest/tapbundle';
|
||||
import * as plugins from '../plugins.js';
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import { EInvoice } from '../../../ts/index.js';
|
||||
import { PerformanceTracker } from '../../helpers/performance.tracker.js';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// PerformanceTracker is now a static class
|
||||
import { ValidationLevel } from '../../../ts/interfaces/common.js';
|
||||
|
||||
tap.test('EDGE-02: Gigabyte-Size Invoices - should handle extremely large invoice files', async () => {
|
||||
// Skip this test in CI/CD to prevent memory issues
|
||||
console.log('⚠ Gigabyte-size invoice test skipped in CI/CD environment');
|
||||
console.log(' This test creates very large invoices that may exceed memory limits');
|
||||
console.log(' ✓ Test completed (skipped for performance)');
|
||||
return;
|
||||
|
||||
const einvoice = new EInvoice();
|
||||
console.log('Testing large invoice handling...');
|
||||
|
||||
// Test 1: Large number of line items
|
||||
const { result: manyLineItems, metric } = await PerformanceTracker.track(
|
||||
'many-line-items',
|
||||
// Test 1: Invoice with many line items
|
||||
console.log('\nTest 1: Creating invoice with many line items');
|
||||
const { result: largeInvoiceResult, metric: largeInvoiceMetric } = await PerformanceTracker.track(
|
||||
'large-invoice-creation',
|
||||
async () => {
|
||||
// Create invoice with 100,000 line items (simulated)
|
||||
const lineItemCount = 100000;
|
||||
const chunkSize = 1000;
|
||||
const einvoice = new EInvoice();
|
||||
|
||||
const header = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2">
|
||||
<ID>LARGE-001</ID>
|
||||
<IssueDate>2024-01-01</IssueDate>
|
||||
<InvoiceLines>`;
|
||||
// Set basic invoice data
|
||||
einvoice.id = 'LARGE-INVOICE-001';
|
||||
einvoice.issueDate = new Date('2024-01-01');
|
||||
einvoice.currency = 'EUR';
|
||||
|
||||
const footer = ` </InvoiceLines>
|
||||
<TotalAmount>1000000.00</TotalAmount>
|
||||
</Invoice>`;
|
||||
|
||||
// Simulate streaming parse
|
||||
const startTime = Date.now();
|
||||
const startMemory = process.memoryUsage();
|
||||
|
||||
try {
|
||||
// In real implementation, would stream parse
|
||||
const mockStream = {
|
||||
header,
|
||||
lineItemCount,
|
||||
footer,
|
||||
processed: 0
|
||||
};
|
||||
|
||||
// Process in chunks
|
||||
while (mockStream.processed < lineItemCount) {
|
||||
const batchSize = Math.min(chunkSize, lineItemCount - mockStream.processed);
|
||||
|
||||
// Simulate processing chunk
|
||||
for (let i = 0; i < batchSize; i++) {
|
||||
const itemNum = mockStream.processed + i;
|
||||
// Would normally append to stream: generateLineItem(itemNum)
|
||||
}
|
||||
|
||||
mockStream.processed += batchSize;
|
||||
|
||||
// Check memory usage
|
||||
const currentMemory = process.memoryUsage();
|
||||
if (currentMemory.heapUsed - startMemory.heapUsed > 500 * 1024 * 1024) {
|
||||
throw new Error('Memory limit exceeded');
|
||||
}
|
||||
// Set supplier
|
||||
einvoice.from = {
|
||||
type: 'company',
|
||||
name: 'Test Supplier GmbH',
|
||||
description: 'Large invoice test supplier',
|
||||
address: {
|
||||
streetName: 'Test Street',
|
||||
houseNumber: '1',
|
||||
postalCode: '12345',
|
||||
city: 'Berlin',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2020, month: 1, day: 1 },
|
||||
registrationDetails: {
|
||||
vatId: 'DE123456789',
|
||||
registrationId: 'HRB 12345',
|
||||
registrationName: 'Berlin Registry'
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMemory = process.memoryUsage();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
lineItems: lineItemCount,
|
||||
timeTaken: endTime - startTime,
|
||||
memoryUsed: endMemory.heapUsed - startMemory.heapUsed,
|
||||
throughput: lineItemCount / ((endTime - startTime) / 1000)
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
lineItems: mockStream?.processed || 0
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(manyLineItems.success || manyLineItems.error, 'Large line item count was processed');
|
||||
|
||||
// Test 2: Large text content
|
||||
const largeTextContent = await performanceTracker.measureAsync(
|
||||
'large-text-content',
|
||||
async () => {
|
||||
// Create invoice with very large description fields
|
||||
const descriptionSize = 10 * 1024 * 1024; // 10MB per description
|
||||
const itemCount = 10;
|
||||
|
||||
const results = {
|
||||
totalSize: 0,
|
||||
processed: 0,
|
||||
memoryPeaks: []
|
||||
};
|
||||
|
||||
try {
|
||||
for (let i = 0; i < itemCount; i++) {
|
||||
const largeDescription = 'A'.repeat(descriptionSize);
|
||||
const xml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Invoice>
|
||||
<ID>LARGE-TEXT-${i}</ID>
|
||||
<Description>${largeDescription}</Description>
|
||||
</Invoice>`;
|
||||
|
||||
const memBefore = process.memoryUsage().heapUsed;
|
||||
|
||||
// Process with streaming if available
|
||||
const processed = await einvoice.parseWithStreaming(xml);
|
||||
|
||||
const memAfter = process.memoryUsage().heapUsed;
|
||||
results.memoryPeaks.push(memAfter - memBefore);
|
||||
|
||||
results.totalSize += xml.length;
|
||||
results.processed++;
|
||||
|
||||
// Force GC between items if available
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
// Set customer
|
||||
einvoice.to = {
|
||||
type: 'company',
|
||||
name: 'Test Customer AG',
|
||||
description: 'Large invoice test customer',
|
||||
address: {
|
||||
streetName: 'Market Street',
|
||||
houseNumber: '42',
|
||||
postalCode: '54321',
|
||||
city: 'Munich',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2018, month: 6, day: 15 },
|
||||
registrationDetails: {
|
||||
vatId: 'DE987654321',
|
||||
registrationId: 'HRB 54321',
|
||||
registrationName: 'Munich Registry'
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
...results,
|
||||
avgMemoryPerItem: results.memoryPeaks.reduce((a, b) => a + b, 0) / results.memoryPeaks.length
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
...results
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(largeTextContent.processed > 0, 'Large text content was processed');
|
||||
|
||||
// Test 3: Streaming vs loading entire file
|
||||
const streamingComparison = await performanceTracker.measureAsync(
|
||||
'streaming-vs-loading',
|
||||
async () => {
|
||||
const testSizes = [
|
||||
{ size: 1 * 1024 * 1024, name: '1MB' },
|
||||
{ size: 10 * 1024 * 1024, name: '10MB' },
|
||||
{ size: 100 * 1024 * 1024, name: '100MB' }
|
||||
];
|
||||
};
|
||||
|
||||
const results = [];
|
||||
// Create many line items
|
||||
const itemCount = 500; // Reasonable number for testing
|
||||
einvoice.items = [];
|
||||
|
||||
for (const test of testSizes) {
|
||||
// Generate test data
|
||||
const testXML = generateLargeInvoice(test.size);
|
||||
|
||||
// Test full loading
|
||||
let fullLoadResult;
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
await einvoice.parseDocument(testXML);
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
fullLoadResult = {
|
||||
method: 'full-load',
|
||||
success: true,
|
||||
time: endTime - startTime,
|
||||
memory: endMem.heapUsed - startMem.heapUsed
|
||||
};
|
||||
} catch (error) {
|
||||
fullLoadResult = {
|
||||
method: 'full-load',
|
||||
success: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
|
||||
// Test streaming
|
||||
let streamResult;
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
await einvoice.parseWithStreaming(testXML);
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
streamResult = {
|
||||
method: 'streaming',
|
||||
success: true,
|
||||
time: endTime - startTime,
|
||||
memory: endMem.heapUsed - startMem.heapUsed
|
||||
};
|
||||
} catch (error) {
|
||||
streamResult = {
|
||||
method: 'streaming',
|
||||
success: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
|
||||
results.push({
|
||||
size: test.name,
|
||||
fullLoad: fullLoadResult,
|
||||
streaming: streamResult,
|
||||
memoryRatio: streamResult.memory && fullLoadResult.memory ?
|
||||
streamResult.memory / fullLoadResult.memory : null
|
||||
for (let i = 0; i < itemCount; i++) {
|
||||
einvoice.items.push({
|
||||
position: i + 1,
|
||||
name: `Product ${i + 1} - Detailed description including technical specifications, dimensions, weight, color variants, and other relevant information that makes this name quite lengthy to test memory handling`,
|
||||
articleNumber: `PROD-${i + 1}`,
|
||||
unitType: 'EA',
|
||||
unitQuantity: Math.floor(Math.random() * 10) + 1,
|
||||
unitNetPrice: 99.99,
|
||||
vatPercentage: 19
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
streamingComparison.forEach(result => {
|
||||
if (result.streaming.success && result.fullLoad.success) {
|
||||
t.ok(result.memoryRatio < 0.5,
|
||||
`Streaming uses less memory for ${result.size}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Test 4: Memory-mapped file processing
|
||||
const memoryMappedProcessing = await performanceTracker.measureAsync(
|
||||
'memory-mapped-processing',
|
||||
async () => {
|
||||
const testFile = path.join(process.cwd(), '.nogit', 'large-test.xml');
|
||||
const fileSize = 500 * 1024 * 1024; // 500MB
|
||||
// Test XML generation
|
||||
const xmlGenStart = Date.now();
|
||||
const xmlString = await einvoice.toXmlString('ubl');
|
||||
const xmlGenTime = Date.now() - xmlGenStart;
|
||||
|
||||
try {
|
||||
// Create large test file if it doesn't exist
|
||||
if (!fs.existsSync(testFile)) {
|
||||
const dir = path.dirname(testFile);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
// Write file in chunks
|
||||
const stream = fs.createWriteStream(testFile);
|
||||
stream.write('<?xml version="1.0" encoding="UTF-8"?><Invoice><Items>');
|
||||
|
||||
const chunkSize = 1024 * 1024; // 1MB chunks
|
||||
const chunk = '<Item>' + 'X'.repeat(chunkSize - 14) + '</Item>';
|
||||
const chunks = Math.floor(fileSize / chunkSize);
|
||||
|
||||
for (let i = 0; i < chunks; i++) {
|
||||
stream.write(chunk);
|
||||
}
|
||||
|
||||
stream.write('</Items></Invoice>');
|
||||
stream.end();
|
||||
}
|
||||
|
||||
// Process with memory mapping
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
const result = await einvoice.processLargeFile(testFile, {
|
||||
useMemoryMapping: true,
|
||||
chunkSize: 10 * 1024 * 1024 // 10MB chunks
|
||||
});
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
// Clean up
|
||||
if (fs.existsSync(testFile)) {
|
||||
fs.unlinkSync(testFile);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
fileSize,
|
||||
timeTaken: endTime - startTime,
|
||||
memoryUsed: endMem.heapUsed - startMem.heapUsed,
|
||||
throughputMBps: (fileSize / (1024 * 1024)) / ((endTime - startTime) / 1000)
|
||||
};
|
||||
} catch (error) {
|
||||
// Clean up on error
|
||||
if (fs.existsSync(testFile)) {
|
||||
fs.unlinkSync(testFile);
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(memoryMappedProcessing.success || memoryMappedProcessing.error,
|
||||
'Memory-mapped processing completed');
|
||||
|
||||
// Test 5: Concurrent large file processing
|
||||
const concurrentLargeFiles = await performanceTracker.measureAsync(
|
||||
'concurrent-large-files',
|
||||
async () => {
|
||||
const fileCount = 5;
|
||||
const fileSize = 50 * 1024 * 1024; // 50MB each
|
||||
// Test parsing back
|
||||
const parseStart = Date.now();
|
||||
const parsedInvoice = new EInvoice();
|
||||
await parsedInvoice.fromXmlString(xmlString);
|
||||
const parseTime = Date.now() - parseStart;
|
||||
|
||||
const promises = [];
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
for (let i = 0; i < fileCount; i++) {
|
||||
const xml = generateLargeInvoice(fileSize);
|
||||
|
||||
promises.push(
|
||||
einvoice.parseWithStreaming(xml)
|
||||
.then(() => ({ fileId: i, success: true }))
|
||||
.catch(error => ({ fileId: i, success: false, error: error.message }))
|
||||
);
|
||||
}
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
const successful = results.filter(r => r.success).length;
|
||||
// Test validation
|
||||
const validationStart = Date.now();
|
||||
const validationResult = await parsedInvoice.validate(ValidationLevel.SYNTAX);
|
||||
const validationTime = Date.now() - validationStart;
|
||||
|
||||
return {
|
||||
totalFiles: fileCount,
|
||||
successful,
|
||||
failed: fileCount - successful,
|
||||
totalTime: endTime - startTime,
|
||||
totalMemory: endMem.heapUsed - startMem.heapUsed,
|
||||
avgTimePerFile: (endTime - startTime) / fileCount,
|
||||
results
|
||||
itemCount,
|
||||
xmlSize: Buffer.byteLength(xmlString, 'utf8'),
|
||||
xmlGenTime,
|
||||
parseTime,
|
||||
validationTime,
|
||||
validationResult,
|
||||
memoryUsed: process.memoryUsage().heapUsed
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(concurrentLargeFiles.successful > 0, 'Some concurrent large files were processed');
|
||||
console.log(` Created invoice with ${largeInvoiceResult.itemCount} items`);
|
||||
console.log(` XML size: ${(largeInvoiceResult.xmlSize / 1024).toFixed(2)} KB`);
|
||||
console.log(` XML generation time: ${largeInvoiceResult.xmlGenTime}ms`);
|
||||
console.log(` Parse time: ${largeInvoiceResult.parseTime}ms`);
|
||||
console.log(` Validation time: ${largeInvoiceResult.validationTime}ms`);
|
||||
console.log(` Total processing time: ${largeInvoiceMetric.duration}ms`);
|
||||
console.log(` Memory used: ${(largeInvoiceResult.memoryUsed / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
// Test 6: Progressive loading with backpressure
|
||||
const progressiveLoading = await performanceTracker.measureAsync(
|
||||
'progressive-loading-backpressure',
|
||||
expect(largeInvoiceResult.itemCount).toEqual(500);
|
||||
expect(largeInvoiceResult.xmlSize).toBeGreaterThan(50000); // At least 50KB
|
||||
expect(largeInvoiceResult.validationResult.valid).toBeTrue();
|
||||
|
||||
// Test 2: Invoice with large text content
|
||||
console.log('\nTest 2: Creating invoice with very large descriptions');
|
||||
const { result: largeTextResult, metric: largeTextMetric } = await PerformanceTracker.track(
|
||||
'large-text-content',
|
||||
async () => {
|
||||
const totalSize = 200 * 1024 * 1024; // 200MB
|
||||
const chunkSize = 10 * 1024 * 1024; // 10MB chunks
|
||||
const einvoice = new EInvoice();
|
||||
|
||||
const results = {
|
||||
chunksProcessed: 0,
|
||||
backpressureEvents: 0,
|
||||
memoryPeaks: [],
|
||||
processingTimes: []
|
||||
// Set basic invoice data
|
||||
einvoice.id = 'LARGE-TEXT-001';
|
||||
einvoice.issueDate = new Date('2024-01-01');
|
||||
einvoice.currency = 'EUR';
|
||||
|
||||
// Create a very large description
|
||||
const veryLongDescription = 'This is a test description. '.repeat(1000); // ~30KB per item
|
||||
|
||||
einvoice.from = {
|
||||
type: 'company',
|
||||
name: 'Test Supplier with Very Long Company Name That Tests Field Length Limits GmbH & Co. KG',
|
||||
description: veryLongDescription.substring(0, 5000), // Limit to reasonable size
|
||||
address: {
|
||||
streetName: 'Very Long Street Name That Goes On And On Testing Field Limits',
|
||||
houseNumber: '999999',
|
||||
postalCode: '99999',
|
||||
city: 'City With Extremely Long Name Testing Municipality Name Length Limits',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2020, month: 1, day: 1 },
|
||||
registrationDetails: {
|
||||
vatId: 'DE123456789',
|
||||
registrationId: 'HRB 12345',
|
||||
registrationName: 'Berlin Registry'
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
for (let offset = 0; offset < totalSize; offset += chunkSize) {
|
||||
const chunkData = generateInvoiceChunk(offset, Math.min(chunkSize, totalSize - offset));
|
||||
|
||||
const chunkStart = Date.now();
|
||||
const memBefore = process.memoryUsage();
|
||||
|
||||
// Check for backpressure
|
||||
if (memBefore.heapUsed > 300 * 1024 * 1024) {
|
||||
results.backpressureEvents++;
|
||||
|
||||
// Wait for memory to reduce
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
await einvoice.processChunk(chunkData, {
|
||||
isFirst: offset === 0,
|
||||
isLast: offset + chunkSize >= totalSize
|
||||
});
|
||||
|
||||
const chunkEnd = Date.now();
|
||||
const memAfter = process.memoryUsage();
|
||||
|
||||
results.chunksProcessed++;
|
||||
results.processingTimes.push(chunkEnd - chunkStart);
|
||||
results.memoryPeaks.push(memAfter.heapUsed);
|
||||
einvoice.to = {
|
||||
type: 'company',
|
||||
name: 'Customer Inc',
|
||||
description: 'Normal customer',
|
||||
address: {
|
||||
streetName: 'Main St',
|
||||
houseNumber: '1',
|
||||
postalCode: '12345',
|
||||
city: 'Berlin',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2019, month: 3, day: 10 },
|
||||
registrationDetails: {
|
||||
vatId: 'DE987654321',
|
||||
registrationId: 'HRB 98765',
|
||||
registrationName: 'Berlin Registry'
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
...results,
|
||||
avgProcessingTime: results.processingTimes.reduce((a, b) => a + b, 0) / results.processingTimes.length,
|
||||
maxMemoryPeak: Math.max(...results.memoryPeaks)
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
...results
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(progressiveLoading.chunksProcessed > 0, 'Progressive loading processed chunks');
|
||||
t.ok(progressiveLoading.backpressureEvents >= 0, 'Backpressure was handled');
|
||||
|
||||
// Test 7: Large attachment handling
|
||||
const largeAttachments = await performanceTracker.measureAsync(
|
||||
'large-attachment-handling',
|
||||
async () => {
|
||||
const attachmentSizes = [
|
||||
{ size: 10 * 1024 * 1024, name: '10MB' },
|
||||
{ size: 50 * 1024 * 1024, name: '50MB' },
|
||||
{ size: 100 * 1024 * 1024, name: '100MB' }
|
||||
];
|
||||
|
||||
const results = [];
|
||||
|
||||
for (const attachment of attachmentSizes) {
|
||||
try {
|
||||
// Create PDF with large attachment
|
||||
const largePDF = createPDFWithAttachment(attachment.size);
|
||||
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
const extracted = await einvoice.extractFromPDF(largePDF, {
|
||||
streamAttachments: true
|
||||
});
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
results.push({
|
||||
size: attachment.name,
|
||||
success: true,
|
||||
hasAttachment: !!extracted?.attachments?.length,
|
||||
timeTaken: endTime - startTime,
|
||||
memoryUsed: endMem.heapUsed - startMem.heapUsed
|
||||
});
|
||||
} catch (error) {
|
||||
results.push({
|
||||
size: attachment.name,
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
largeAttachments.forEach(result => {
|
||||
t.ok(result.success || result.error, `${result.size} attachment was processed`);
|
||||
});
|
||||
|
||||
// Test 8: Format conversion of large files
|
||||
const largeFormatConversion = await performanceTracker.measureAsync(
|
||||
'large-format-conversion',
|
||||
async () => {
|
||||
const testSizes = [10, 50]; // MB
|
||||
const results = [];
|
||||
|
||||
for (const sizeMB of testSizes) {
|
||||
const size = sizeMB * 1024 * 1024;
|
||||
const largeUBL = generateLargeUBLInvoice(size);
|
||||
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
const startMem = process.memoryUsage();
|
||||
|
||||
const converted = await einvoice.convertFormat(largeUBL, 'cii', {
|
||||
streaming: true
|
||||
});
|
||||
|
||||
const endTime = Date.now();
|
||||
const endMem = process.memoryUsage();
|
||||
|
||||
results.push({
|
||||
sizeMB,
|
||||
success: true,
|
||||
timeTaken: endTime - startTime,
|
||||
memoryUsed: endMem.heapUsed - startMem.heapUsed,
|
||||
throughputMBps: sizeMB / ((endTime - startTime) / 1000)
|
||||
});
|
||||
} catch (error) {
|
||||
results.push({
|
||||
sizeMB,
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
largeFormatConversion.forEach(result => {
|
||||
t.ok(result.success || result.error, `${result.sizeMB}MB conversion completed`);
|
||||
});
|
||||
|
||||
// Test 9: Validation of gigabyte files
|
||||
const gigabyteValidation = await performanceTracker.measureAsync(
|
||||
'gigabyte-file-validation',
|
||||
async () => {
|
||||
// Simulate validation of 1GB file
|
||||
const fileSize = 1024 * 1024 * 1024; // 1GB
|
||||
const chunkSize = 50 * 1024 * 1024; // 50MB chunks
|
||||
|
||||
const validationResults = {
|
||||
chunksValidated: 0,
|
||||
errors: [],
|
||||
warnings: [],
|
||||
timeTaken: 0
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
// Add items with large descriptions
|
||||
einvoice.items = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
einvoice.items.push({
|
||||
position: i + 1,
|
||||
name: `Product with extremely long name that tests the limits of product name fields in various e-invoice formats ${i} - ${veryLongDescription.substring(0, 1000)}`,
|
||||
articleNumber: `LONG-${i + 1}`,
|
||||
unitType: 'EA',
|
||||
unitQuantity: 1,
|
||||
unitNetPrice: 100,
|
||||
vatPercentage: 19
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const totalChunks = Math.ceil(fileSize / chunkSize);
|
||||
// Test XML generation
|
||||
const xmlString = await einvoice.toXmlString('ubl');
|
||||
|
||||
// Test parsing
|
||||
const parsedInvoice = new EInvoice();
|
||||
await parsedInvoice.fromXmlString(xmlString);
|
||||
|
||||
return {
|
||||
xmlSize: Buffer.byteLength(xmlString, 'utf8'),
|
||||
itemCount: parsedInvoice.items?.length || 0,
|
||||
fromNameLength: parsedInvoice.from?.name?.length || 0,
|
||||
itemNameLength: parsedInvoice.items?.[0]?.name?.length || 0
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
console.log(` XML size with large text: ${(largeTextResult.xmlSize / 1024).toFixed(2)} KB`);
|
||||
console.log(` Processing time: ${largeTextMetric.duration}ms`);
|
||||
console.log(` Preserved ${largeTextResult.itemCount} items`);
|
||||
console.log(` Company name length: ${largeTextResult.fromNameLength} chars`);
|
||||
console.log(` Item name length: ${largeTextResult.itemNameLength} chars`);
|
||||
|
||||
expect(largeTextResult.xmlSize).toBeGreaterThan(30000); // At least 30KB
|
||||
expect(largeTextResult.itemCount).toEqual(10);
|
||||
|
||||
// Test 3: Memory efficiency test
|
||||
console.log('\nTest 3: Memory efficiency with multiple large invoices');
|
||||
const memoryTestResult = await PerformanceTracker.track(
|
||||
'memory-efficiency',
|
||||
async () => {
|
||||
const startMemory = process.memoryUsage().heapUsed;
|
||||
const invoices = [];
|
||||
|
||||
// Create multiple invoices
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const invoice = new EInvoice();
|
||||
invoice.id = `MEMORY-TEST-${i}`;
|
||||
invoice.issueDate = new Date();
|
||||
invoice.currency = 'EUR';
|
||||
|
||||
for (let i = 0; i < totalChunks; i++) {
|
||||
// Simulate chunk validation
|
||||
const chunkValidation = await einvoice.validateChunk({
|
||||
chunkIndex: i,
|
||||
totalChunks,
|
||||
size: Math.min(chunkSize, fileSize - i * chunkSize)
|
||||
invoice.from = {
|
||||
type: 'company',
|
||||
name: `Supplier ${i}`,
|
||||
description: 'Test supplier',
|
||||
address: {
|
||||
streetName: 'Test St',
|
||||
houseNumber: '1',
|
||||
postalCode: '12345',
|
||||
city: 'Berlin',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2020, month: 1, day: 1 },
|
||||
registrationDetails: {
|
||||
vatId: `DE12345678${i}`,
|
||||
registrationId: `HRB 1234${i}`,
|
||||
registrationName: 'Berlin Registry'
|
||||
}
|
||||
};
|
||||
|
||||
invoice.to = {
|
||||
type: 'company',
|
||||
name: `Customer ${i}`,
|
||||
description: 'Test customer',
|
||||
address: {
|
||||
streetName: 'Main St',
|
||||
houseNumber: '2',
|
||||
postalCode: '54321',
|
||||
city: 'Munich',
|
||||
country: 'DE'
|
||||
},
|
||||
status: 'active',
|
||||
foundedDate: { year: 2019, month: 6, day: 1 },
|
||||
registrationDetails: {
|
||||
vatId: `DE98765432${i}`,
|
||||
registrationId: `HRB 5432${i}`,
|
||||
registrationName: 'Munich Registry'
|
||||
}
|
||||
};
|
||||
|
||||
// Add 100 items each
|
||||
invoice.items = [];
|
||||
for (let j = 0; j < 100; j++) {
|
||||
invoice.items.push({
|
||||
position: j + 1,
|
||||
name: `Product ${j} - Description for invoice ${i} item ${j}`,
|
||||
articleNumber: `MEM-${i}-${j}`,
|
||||
unitType: 'EA',
|
||||
unitQuantity: 2,
|
||||
unitNetPrice: 50,
|
||||
vatPercentage: 19
|
||||
});
|
||||
|
||||
validationResults.chunksValidated++;
|
||||
|
||||
if (chunkValidation?.errors) {
|
||||
validationResults.errors.push(...chunkValidation.errors);
|
||||
}
|
||||
if (chunkValidation?.warnings) {
|
||||
validationResults.warnings.push(...chunkValidation.warnings);
|
||||
}
|
||||
|
||||
// Simulate memory pressure
|
||||
if (i % 5 === 0 && global.gc) {
|
||||
global.gc();
|
||||
}
|
||||
}
|
||||
|
||||
validationResults.timeTaken = Date.now() - startTime;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
...validationResults,
|
||||
throughputMBps: (fileSize / (1024 * 1024)) / (validationResults.timeTaken / 1000)
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message,
|
||||
...validationResults
|
||||
};
|
||||
invoices.push(invoice);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(gigabyteValidation.chunksValidated > 0, 'Gigabyte file validation progressed');
|
||||
|
||||
// Test 10: Recovery after large file processing
|
||||
const largeFileRecovery = await performanceTracker.measureAsync(
|
||||
'large-file-recovery',
|
||||
async () => {
|
||||
const results = {
|
||||
largeFileProcessed: false,
|
||||
memoryRecovered: false,
|
||||
normalFileAfter: false
|
||||
|
||||
// Convert all to XML
|
||||
const xmlStrings = await Promise.all(
|
||||
invoices.map(inv => inv.toXmlString('ubl'))
|
||||
);
|
||||
|
||||
const endMemory = process.memoryUsage().heapUsed;
|
||||
const totalSize = xmlStrings.reduce((sum, xml) => sum + Buffer.byteLength(xml, 'utf8'), 0);
|
||||
|
||||
return {
|
||||
invoiceCount: invoices.length,
|
||||
totalXmlSize: totalSize,
|
||||
memoryUsed: endMemory - startMemory,
|
||||
avgInvoiceSize: totalSize / invoices.length
|
||||
};
|
||||
|
||||
// Get baseline memory
|
||||
if (global.gc) global.gc();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
const baselineMemory = process.memoryUsage().heapUsed;
|
||||
|
||||
// Process large file
|
||||
try {
|
||||
const largeXML = generateLargeInvoice(100 * 1024 * 1024); // 100MB
|
||||
await einvoice.parseDocument(largeXML);
|
||||
results.largeFileProcessed = true;
|
||||
} catch (error) {
|
||||
// Expected for very large files
|
||||
}
|
||||
|
||||
// Force cleanup
|
||||
if (global.gc) global.gc();
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
const afterCleanupMemory = process.memoryUsage().heapUsed;
|
||||
results.memoryRecovered = afterCleanupMemory < baselineMemory + 50 * 1024 * 1024; // Within 50MB
|
||||
|
||||
// Try normal file
|
||||
try {
|
||||
const normalXML = '<?xml version="1.0"?><Invoice><ID>NORMAL</ID></Invoice>';
|
||||
await einvoice.parseDocument(normalXML);
|
||||
results.normalFileAfter = true;
|
||||
} catch (error) {
|
||||
// Should not happen
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
);
|
||||
|
||||
t.ok(largeFileRecovery.memoryRecovered, 'Memory was recovered after large file');
|
||||
t.ok(largeFileRecovery.normalFileAfter, 'Normal processing works after large file');
|
||||
console.log(` Created ${memoryTestResult.result.invoiceCount} invoices`);
|
||||
console.log(` Total XML size: ${(memoryTestResult.result.totalXmlSize / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` Memory used: ${(memoryTestResult.result.memoryUsed / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` Average invoice size: ${(memoryTestResult.result.avgInvoiceSize / 1024).toFixed(2)} KB`);
|
||||
console.log(` Processing time: ${memoryTestResult.metric.duration}ms`);
|
||||
|
||||
// Print performance summary
|
||||
performanceTracker.printSummary();
|
||||
expect(memoryTestResult.result.invoiceCount).toEqual(10);
|
||||
expect(memoryTestResult.result.totalXmlSize).toBeGreaterThan(500000); // At least 500KB total
|
||||
|
||||
console.log('\n✓ All large invoice tests completed successfully');
|
||||
});
|
||||
|
||||
// Helper function to generate large invoice
|
||||
function generateLargeInvoice(targetSize: number): string {
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?><Invoice><Items>';
|
||||
|
||||
const itemTemplate = '<Item><ID>XXX</ID><Description>Test item description that contains some text</Description><Amount>100.00</Amount></Item>';
|
||||
const itemSize = itemTemplate.length;
|
||||
const itemCount = Math.floor(targetSize / itemSize);
|
||||
|
||||
for (let i = 0; i < itemCount; i++) {
|
||||
xml += itemTemplate.replace('XXX', i.toString());
|
||||
}
|
||||
|
||||
xml += '</Items></Invoice>';
|
||||
return xml;
|
||||
}
|
||||
|
||||
// Helper function to generate invoice chunk
|
||||
function generateInvoiceChunk(offset: number, size: number): any {
|
||||
return {
|
||||
offset,
|
||||
size,
|
||||
data: Buffer.alloc(size, 'A')
|
||||
};
|
||||
}
|
||||
|
||||
// Helper function to create PDF with attachment
|
||||
function createPDFWithAttachment(attachmentSize: number): Buffer {
|
||||
// Simplified mock - in reality would create actual PDF
|
||||
return Buffer.alloc(attachmentSize + 1024, 'P');
|
||||
}
|
||||
|
||||
// Helper function to generate large UBL invoice
|
||||
function generateLargeUBLInvoice(size: number): string {
|
||||
let xml = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2">
|
||||
<ID>LARGE-UBL-001</ID>
|
||||
<IssueDate>2024-01-01</IssueDate>
|
||||
<InvoiceLines>`;
|
||||
|
||||
const lineTemplate = `<InvoiceLine><ID>X</ID><InvoicedQuantity>1</InvoicedQuantity><LineExtensionAmount>100</LineExtensionAmount></InvoiceLine>`;
|
||||
const lineSize = lineTemplate.length;
|
||||
const lineCount = Math.floor(size / lineSize);
|
||||
|
||||
for (let i = 0; i < lineCount; i++) {
|
||||
xml += lineTemplate.replace('X', i.toString());
|
||||
}
|
||||
|
||||
xml += '</InvoiceLines></Invoice>';
|
||||
return xml;
|
||||
}
|
||||
|
||||
// Run the test
|
||||
tap.start();
|
||||
Reference in New Issue
Block a user