diff --git a/changelog.md b/changelog.md index 7e6996b..a51455f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,14 @@ # Changelog +## 2026-01-19 - 1.14.1 - fix(extraction) +improve JSON extraction prompts and model options for invoice and bank statement tests + +- Refactor JSON extraction prompts to be sent after the document text and add explicit 'WHERE TO FIND DATA' and 'RULES' sections for clearer extraction guidance +- Change chat message flow to: send document, assistant acknowledgement, then the JSON extraction prompt (avoids concatenating large prompts into one message) +- Add model options (num_ctx: 32768, temperature: 0) to give larger context windows and deterministic JSON output +- Simplify logging to avoid printing full prompt contents; log document and prompt lengths instead +- Increase timeouts for large documents to 600000ms (10 minutes) where applicable + ## 2026-01-19 - 1.14.0 - feat(docker-images) add vLLM-based Nanonets-OCR2-3B image, Qwen3-VL Ollama image and refactor build/docs/tests to use new runtime/layout diff --git a/test/test.bankstatements.nanonets.ts b/test/test.bankstatements.nanonets.ts index 9c8ef81..b3254fc 100644 --- a/test/test.bankstatements.nanonets.ts +++ b/test/test.bankstatements.nanonets.ts @@ -51,11 +51,21 @@ If there is an image in the document and image caption is not present, add a sma Watermarks should be wrapped in brackets. Ex: OFFICIAL COPY. Page numbers should be wrapped in brackets. Ex: 14.`; -// JSON extraction prompt for GPT-OSS 20B -const JSON_EXTRACTION_PROMPT = `Extract ALL transactions from this bank statement as JSON array. Each transaction: {"date": "YYYY-MM-DD", "counterparty": "NAME", "amount": -25.99}. Amount negative for debits, positive for credits. Only include actual transactions, not balances. Return ONLY JSON array, no explanation. +// JSON extraction prompt for GPT-OSS 20B (sent AFTER the statement text is provided) +const JSON_EXTRACTION_PROMPT = `Extract ALL transactions from the bank statement. Return ONLY valid JSON array. -STATEMENT: -`; +WHERE TO FIND DATA: +- Transactions are typically in TABLES with columns: Date, Description/Counterparty, Debit, Credit, Balance +- Look for rows with actual money movements, NOT header rows or summary totals + +RULES: +1. date: Convert to YYYY-MM-DD format +2. counterparty: The name/description of who the money went to/from +3. amount: NEGATIVE for debits/withdrawals, POSITIVE for credits/deposits +4. Only include actual transactions, NOT opening/closing balances + +JSON array only: +[{"date":"YYYY-MM-DD","counterparty":"NAME","amount":-25.99}]`; // Constants for smart batching const MAX_VISUAL_TOKENS = 28000; // ~32K context minus prompt/output headroom @@ -246,12 +256,8 @@ async function ensureExtractionModel(): Promise { */ async function extractTransactionsFromMarkdown(markdown: string, queryId: string): Promise { const startTime = Date.now(); - const fullPrompt = JSON_EXTRACTION_PROMPT + markdown; - // Log exact prompt - console.log(`\n [${queryId}] ===== PROMPT =====`); - console.log(fullPrompt); - console.log(` [${queryId}] ===== END PROMPT (${fullPrompt.length} chars) =====\n`); + console.log(` [${queryId}] Statement: ${markdown.length} chars, Prompt: ${JSON_EXTRACTION_PROMPT.length} chars`); const response = await fetch(`${OLLAMA_URL}/api/chat`, { method: 'POST', @@ -261,9 +267,15 @@ async function extractTransactionsFromMarkdown(markdown: string, queryId: string messages: [ { role: 'user', content: 'Hi there, how are you?' }, { role: 'assistant', content: 'Good, how can I help you today?' }, - { role: 'user', content: fullPrompt }, + { role: 'user', content: `Here is a bank statement document:\n\n${markdown}` }, + { role: 'assistant', content: 'I have read the bank statement document you provided. I can see all the transaction data. What would you like me to do with it?' }, + { role: 'user', content: JSON_EXTRACTION_PROMPT }, ], stream: true, + options: { + num_ctx: 32768, // Larger context for long statements + thinking + temperature: 0, // Deterministic for JSON extraction + }, }), signal: AbortSignal.timeout(600000), // 10 minute timeout }); diff --git a/test/test.invoices.extraction.ts b/test/test.invoices.extraction.ts index 7f31558..d581542 100644 --- a/test/test.invoices.extraction.ts +++ b/test/test.invoices.extraction.ts @@ -197,6 +197,10 @@ async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Pr { role: 'user', content: JSON_EXTRACTION_PROMPT }, ], stream: true, + options: { + num_ctx: 32768, // Larger context for long invoices + thinking + temperature: 0, // Deterministic for JSON extraction + }, }), signal: AbortSignal.timeout(120000), // 2 min timeout }); diff --git a/test/test.invoices.nanonets.ts b/test/test.invoices.nanonets.ts index f3cf306..cbe42c9 100644 --- a/test/test.invoices.nanonets.ts +++ b/test/test.invoices.nanonets.ts @@ -54,31 +54,24 @@ If there is an image in the document and image caption is not present, add a sma Watermarks should be wrapped in brackets. Ex: OFFICIAL COPY. Page numbers should be wrapped in brackets. Ex: 14.`; -// JSON extraction prompt for GPT-OSS 20B -const JSON_EXTRACTION_PROMPT = `You are an invoice data extractor. Below is an invoice document converted to text/markdown. Extract the key invoice fields as JSON. +// JSON extraction prompt for GPT-OSS 20B (sent AFTER the invoice text is provided) +const JSON_EXTRACTION_PROMPT = `Extract key fields from the invoice. Return ONLY valid JSON. -IMPORTANT RULES: -1. invoice_number: The unique invoice/document number (NOT VAT ID, NOT customer ID) -2. invoice_date: Format as YYYY-MM-DD -3. vendor_name: The company that issued the invoice +WHERE TO FIND DATA: +- invoice_number, invoice_date, vendor_name: Look in the HEADER section at the TOP of PAGE 1 (near "Invoice no.", "Invoice date:", "Rechnungsnummer") +- net_amount, vat_amount, total_amount: Look in the SUMMARY section at the BOTTOM (look for "Total", "Amount due", "Gesamtbetrag") + +RULES: +1. invoice_number: Extract ONLY the value (e.g., "R0015632540"), NOT the label "Invoice no." +2. invoice_date: Convert to YYYY-MM-DD format (e.g., "14/04/2022" → "2022-04-14") +3. vendor_name: The company issuing the invoice 4. currency: EUR, USD, or GBP -5. net_amount: Amount before tax -6. vat_amount: Tax/VAT amount -7. total_amount: Final total (gross amount) +5. net_amount: Total before tax +6. vat_amount: Tax amount +7. total_amount: Final total with tax -Return ONLY this JSON format, no explanation: -{ - "invoice_number": "INV-2024-001", - "invoice_date": "2024-01-15", - "vendor_name": "Company Name", - "currency": "EUR", - "net_amount": 100.00, - "vat_amount": 19.00, - "total_amount": 119.00 -} - -INVOICE TEXT: -`; +JSON only: +{"invoice_number":"X","invoice_date":"YYYY-MM-DD","vendor_name":"X","currency":"EUR","net_amount":0,"vat_amount":0,"total_amount":0}`; // Constants for smart batching const MAX_VISUAL_TOKENS = 28000; // ~32K context minus prompt/output headroom @@ -370,12 +363,8 @@ function parseJsonToInvoice(response: string): IInvoice | null { */ async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Promise { const startTime = Date.now(); - const fullPrompt = JSON_EXTRACTION_PROMPT + markdown; - // Log exact prompt - console.log(`\n [${queryId}] ===== PROMPT =====`); - console.log(fullPrompt); - console.log(` [${queryId}] ===== END PROMPT (${fullPrompt.length} chars) =====\n`); + console.log(` [${queryId}] Invoice: ${markdown.length} chars, Prompt: ${JSON_EXTRACTION_PROMPT.length} chars`); const response = await fetch(`${OLLAMA_URL}/api/chat`, { method: 'POST', @@ -385,9 +374,15 @@ async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Pr messages: [ { role: 'user', content: 'Hi there, how are you?' }, { role: 'assistant', content: 'Good, how can I help you today?' }, - { role: 'user', content: fullPrompt }, + { role: 'user', content: `Here is an invoice document:\n\n${markdown}` }, + { role: 'assistant', content: 'I have read the invoice document you provided. I can see all the text content. What would you like me to do with it?' }, + { role: 'user', content: JSON_EXTRACTION_PROMPT }, ], stream: true, + options: { + num_ctx: 32768, // Larger context for long invoices + thinking + temperature: 0, // Deterministic for JSON extraction + }, }), signal: AbortSignal.timeout(600000), // 10 minute timeout for large documents });