einvoice/test/suite/einvoice_parsing/test.parse-06.streaming-parse.ts

import { expect, tap } from '@git.zone/tstest/tapbundle';
import * as einvoice from '../../../ts/index.js';
import * as plugins from '../../plugins.js';

tap.test('PARSE-06: Memory-efficient parsing strategies', async () => {
  console.log('Testing memory-efficient parsing of large e-invoices...\n');
  
  // Generate different sized test documents
  const generateLargeInvoice = (lineItems: number): string => {
    const lines = [];
    for (let i = 1; i <= lineItems; i++) {
      lines.push(`
    <cac:InvoiceLine>
      <cbc:ID>${i}</cbc:ID>
      <cbc:InvoicedQuantity unitCode="EA">${i}</cbc:InvoicedQuantity>
      <cbc:LineExtensionAmount currencyID="EUR">${(i * 10).toFixed(2)}</cbc:LineExtensionAmount>
      <cac:Item>
        <cbc:Name>Product Item ${i}</cbc:Name>
        <cbc:Description>Product Item ${i} with a reasonably long description to increase document size for streaming test purposes</cbc:Description>
      </cac:Item>
      <cac:Price>
        <cbc:PriceAmount currencyID="EUR">${(Math.random() * 100).toFixed(2)}</cbc:PriceAmount>
      </cac:Price>
    </cac:InvoiceLine>`);
    }
    
    return `<?xml version="1.0" encoding="UTF-8"?>
<ubl:Invoice xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"
             xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2"
             xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2">
  <cbc:ID>LARGE-${lineItems}</cbc:ID>
  <cbc:IssueDate>2024-01-01</cbc:IssueDate>
  <cac:AccountingSupplierParty>
    <cac:Party>
      <cac:PartyName>
        <cbc:Name>Large Invoice Supplier</cbc:Name>
      </cac:PartyName>
    </cac:Party>
  </cac:AccountingSupplierParty>
  <cac:AccountingCustomerParty>
    <cac:Party>
      <cac:PartyName>
        <cbc:Name>Large Invoice Customer</cbc:Name>
      </cac:PartyName>
    </cac:Party>
  </cac:AccountingCustomerParty>
${lines.join('')}
</ubl:Invoice>`;
  };
  
  const testSizes = [
    { items: 100, expectedSize: '~50KB' },
    { items: 1000, expectedSize: '~500KB' },
    { items: 5000, expectedSize: '~2.5MB' }
  ];
  
  for (const test of testSizes) {
    const startTime = Date.now();
    const startMemory = process.memoryUsage();
    
    const largeXml = generateLargeInvoice(test.items);
    const xmlSize = Buffer.byteLength(largeXml, 'utf8');
    
    console.log(`\nTesting ${test.items} line items (${test.expectedSize}, actual: ${(xmlSize/1024).toFixed(1)}KB):`);
    
    try {
      const invoice = new einvoice.EInvoice();
      await invoice.fromXmlString(largeXml);
      
      const endMemory = process.memoryUsage();
      const memoryDelta = {
        heapUsed: (endMemory.heapUsed - startMemory.heapUsed) / 1024 / 1024,
        external: (endMemory.external - startMemory.external) / 1024 / 1024
      };
      
      const parseTime = Date.now() - startTime;
      
      console.log(`  Parse time: ${parseTime}ms`);
      console.log(`  Memory delta: ${memoryDelta.heapUsed.toFixed(2)}MB heap, ${memoryDelta.external.toFixed(2)}MB external`);
      console.log(`  Parse rate: ${(xmlSize / parseTime * 1000 / 1024 / 1024).toFixed(2)}MB/s`);
      
      // Check if memory usage is reasonable
      const memoryRatio = memoryDelta.heapUsed / (xmlSize / 1024 / 1024);
      console.log(`  Memory ratio: ${memoryRatio.toFixed(2)}x document size`);
      
      if (memoryRatio > 10) {
        console.log('  ⚠️  High memory usage detected');
      } else {
        console.log('  ✓ Memory usage acceptable');
      }
      
      // Verify the invoice was parsed correctly
      expect(invoice.id).toEqual(`LARGE-${test.items}`);
      expect(invoice.items?.length).toEqual(test.items);
      
    } catch (error) {
      console.log(`  ✗ Parse error: ${error.message}`);
    }
    
    // Force garbage collection if available
    if (global.gc) {
      global.gc();
    }
  }
});

tap.test('PARSE-06: Streaming parse simulation', async () => {
  console.log('\nTesting streaming parse behavior...\n');
  
  // Test parsing in chunks (simulating streaming)
  const chunkTests = [
    {
      name: 'Parse partial invoice (incomplete)',
      xml: `<?xml version="1.0"?>
<ubl:Invoice xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2">
  <cbc:ID>PARTIAL-001</cbc:ID>
  <!-- Invoice is incomplete -->`,
      expectError: true
    },
    {
      name: 'Parse complete minimal invoice',
      xml: `<?xml version="1.0"?>
<ubl:Invoice xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"
             xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2">
  <cbc:ID>MINIMAL-001</cbc:ID>
</ubl:Invoice>`,
      expectError: false
    }
  ];
  
  for (const test of chunkTests) {
    console.log(`${test.name}:`);
    
    try {
      const invoice = new einvoice.EInvoice();
      await invoice.fromXmlString(test.xml);
      
      if (test.expectError) {
        console.log('  ✗ Expected error but parsed successfully');
      } else {
        console.log('  ✓ Parsed successfully');
        console.log(`  ID: ${invoice.id}`);
      }
    } catch (error) {
      if (test.expectError) {
        console.log('  ✓ Expected error occurred');
        console.log(`  Error: ${error.message}`);
      } else {
        console.log(`  ✗ Unexpected error: ${error.message}`);
      }
    }
  }
});

tap.test('PARSE-06: Progressive parsing performance', async () => {
  console.log('\nTesting progressive parsing performance...\n');
  
  // Test parsing increasingly complex documents
  const complexityLevels = [
    { name: 'Simple', lineItems: 10, additionalElements: 0 },
    { name: 'Moderate', lineItems: 50, additionalElements: 10 },
    { name: 'Complex', lineItems: 100, additionalElements: 20 },
    { name: 'Very Complex', lineItems: 500, additionalElements: 50 }
  ];
  
  const results = [];
  
  for (const level of complexityLevels) {
    const invoice = `<?xml version="1.0" encoding="UTF-8"?>
<ubl:Invoice xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"
             xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2"
             xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2">
  <cbc:ID>${level.name}-INVOICE</cbc:ID>
  <cbc:IssueDate>2024-01-01</cbc:IssueDate>
  <cbc:DueDate>2024-02-01</cbc:DueDate>
  ${Array.from({length: level.additionalElements}, (_, i) => `
  <cbc:Note>Additional note ${i + 1} for complexity testing</cbc:Note>`).join('')}
  <cac:AccountingSupplierParty>
    <cac:Party>
      <cac:PartyName>
        <cbc:Name>Complex Supplier</cbc:Name>
      </cac:PartyName>
    </cac:Party>
  </cac:AccountingSupplierParty>
  ${Array.from({length: level.lineItems}, (_, i) => `
  <cac:InvoiceLine>
    <cbc:ID>${i + 1}</cbc:ID>
    <cbc:InvoicedQuantity unitCode="EA">1</cbc:InvoicedQuantity>
    <cbc:LineExtensionAmount currencyID="EUR">100.00</cbc:LineExtensionAmount>
    <cac:Item>
      <cbc:Name>Item ${i + 1}</cbc:Name>
    </cac:Item>
  </cac:InvoiceLine>`).join('')}
</ubl:Invoice>`;
    
    const startTime = Date.now();
    const xmlSize = Buffer.byteLength(invoice, 'utf8');
    
    try {
      const einvoiceObj = new einvoice.EInvoice();
      await einvoiceObj.fromXmlString(invoice);
      
      const parseTime = Date.now() - startTime;
      const parseRate = (xmlSize / parseTime * 1000 / 1024).toFixed(2);
      
      results.push({
        level: level.name,
        size: xmlSize,
        time: parseTime,
        rate: parseRate
      });
      
      console.log(`${level.name} (${level.lineItems} items, ${(xmlSize/1024).toFixed(1)}KB):`);
      console.log(`  ✓ Parsed in ${parseTime}ms (${parseRate}KB/s)`);
      
    } catch (error) {
      console.log(`${level.name}: ✗ Error - ${error.message}`);
    }
  }
  
  // Performance summary
  console.log('\nPerformance Summary:');
  results.forEach(r => {
    console.log(`  ${r.level}: ${r.time}ms for ${(r.size/1024).toFixed(1)}KB (${r.rate}KB/s)`);
  });
});

tap.test('PARSE-06: Memory cleanup verification', async () => {
  console.log('\nTesting memory cleanup after parsing...\n');
  
  // Parse a large document and verify memory is released
  const largeXml = `<?xml version="1.0" encoding="UTF-8"?>
<ubl:Invoice xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"
             xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2"
             xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2">
  <cbc:ID>MEMORY-TEST</cbc:ID>
  <cbc:IssueDate>2024-01-01</cbc:IssueDate>
  ${Array.from({length: 1000}, (_, i) => `
  <cac:InvoiceLine>
    <cbc:ID>${i + 1}</cbc:ID>
    <cbc:InvoicedQuantity unitCode="EA">1</cbc:InvoicedQuantity>
    <cbc:LineExtensionAmount currencyID="EUR">100.00</cbc:LineExtensionAmount>
    <cac:Item>
      <cbc:Name>Memory test item ${i + 1} with additional description</cbc:Name>
    </cac:Item>
  </cac:InvoiceLine>`).join('')}
</ubl:Invoice>`;
  
  // Initial memory
  if (global.gc) global.gc();
  const initialMemory = process.memoryUsage().heapUsed;
  
  // Parse multiple times
  console.log('Parsing 5 large invoices sequentially...');
  for (let i = 0; i < 5; i++) {
    const invoice = new einvoice.EInvoice();
    await invoice.fromXmlString(largeXml);
    console.log(`  Parse ${i + 1} complete`);
  }
  
  // Force GC and check memory
  if (global.gc) {
    global.gc();
    await new Promise(resolve => setTimeout(resolve, 100));
    
    const finalMemory = process.memoryUsage().heapUsed;
    const memoryIncrease = (finalMemory - initialMemory) / 1024 / 1024;
    
    console.log(`\nMemory increase after 5 parses: ${memoryIncrease.toFixed(2)}MB`);
    
    if (memoryIncrease > 50) {
      console.log('⚠️  Possible memory leak detected');
    } else {
      console.log('✓ Memory usage within acceptable range');
    }
  } else {
    console.log('⚠️  Manual GC not available - memory leak test skipped');
  }
});

// Run the tests
tap.start();