Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9f9ec0a671 | |||
| 3780105c6f | |||
| d237ad19f4 | |||
| 7652a2df52 | |||
| b316d98f24 | |||
| f0d88fcbe0 |
26
Dockerfile_qwen3vl
Normal file
26
Dockerfile_qwen3vl
Normal file
@@ -0,0 +1,26 @@
|
||||
# Qwen3-VL-30B-A3B Vision Language Model
|
||||
# Q4_K_M quantization (~20GB model)
|
||||
#
|
||||
# Most powerful Qwen vision model:
|
||||
# - 256K context (expandable to 1M)
|
||||
# - Visual agent capabilities
|
||||
# - Code generation from images
|
||||
#
|
||||
# Build: docker build -f Dockerfile_qwen3vl -t qwen3vl .
|
||||
# Run: docker run --gpus all -p 11434:11434 -v ht-ollama-models:/root/.ollama qwen3vl
|
||||
|
||||
FROM ollama/ollama:latest
|
||||
|
||||
# Pre-pull the model during build (optional - can also pull at runtime)
|
||||
# This makes the image larger but faster to start
|
||||
# RUN ollama serve & sleep 5 && ollama pull qwen3-vl:30b-a3b && pkill ollama
|
||||
|
||||
# Expose Ollama API port
|
||||
EXPOSE 11434
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
|
||||
CMD curl -f http://localhost:11434/api/tags || exit 1
|
||||
|
||||
# Start Ollama server
|
||||
CMD ["serve"]
|
||||
27
changelog.md
27
changelog.md
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-01-18 - 1.10.0 - feat(vision)
|
||||
add Qwen3-VL vision model support with Dockerfile and tests; improve invoice OCR conversion and prompts; simplify extraction flow by removing consensus voting
|
||||
|
||||
- Add Dockerfile_qwen3vl to provide an Ollama-based image for Qwen3-VL and expose the Ollama API on port 11434
|
||||
- Introduce test/test.invoices.qwen3vl.ts and ensureQwen3Vl() helper to pull and test qwen3-vl:8b
|
||||
- Improve PDF->PNG conversion and prompt in ministral3 tests (higher DPI, max quality, sharpen) and increase num_predict from 512 to 1024
|
||||
- Simplify extraction pipeline: remove consensus voting, log single-pass results, and simplify OCR HTML sanitization/truncation logic
|
||||
|
||||
## 2026-01-18 - 1.9.0 - feat(tests)
|
||||
add Ministral 3 vision tests and improve invoice extraction pipeline to use Ollama chat schema, sanitization, and multi-page support
|
||||
|
||||
- Add new vision-based test suites for Ministral 3: test/test.invoices.ministral3.ts and test/test.bankstatements.ministral3.ts (model ministral-3:8b).
|
||||
- Introduce ensureMinistral3() helper to start/check Ollama/MiniCPM model in test/helpers/docker.ts.
|
||||
- Switch invoice extraction to use Ollama /api/chat with a JSON schema (format) and streaming support (reads message.content).
|
||||
- Improve HTML handling: sanitizeHtml() to remove OCR artifacts, concatenate multi-page HTML with page markers, and increase truncation limits.
|
||||
- Enhance response parsing: strip Markdown code fences, robustly locate JSON object boundaries, and provide clearer JSON parse errors.
|
||||
- Add PDF->PNG conversion (ImageMagick) and direct image-based extraction flow for vision model tests.
|
||||
|
||||
## 2026-01-18 - 1.8.0 - feat(paddleocr-vl)
|
||||
add structured HTML output and table parsing for PaddleOCR-VL, update API, tests, and README
|
||||
|
||||
- Add result_to_html(), parse_markdown_table(), and parse_paddleocr_table() to emit semantic HTML and convert OCR/markdown tables to proper <table> elements
|
||||
- Enhance result_to_markdown() with positional/type hints (header/footer/title/table/figure) to improve downstream LLM processing
|
||||
- Expose 'html' in supported formats and handle output_format='html' in parse endpoints and CLI flow
|
||||
- Update tests to request HTML output and extract invoice fields from structured HTML (test/test.invoices.paddleocr-vl.ts)
|
||||
- Refresh README with usage, new images/tags, architecture notes, and troubleshooting for the updated pipeline
|
||||
|
||||
## 2026-01-17 - 1.7.1 - fix(docker)
|
||||
standardize Dockerfile and entrypoint filenames; add GPU-specific Dockerfiles and update build and test references
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ Provides REST API for document parsing using:
|
||||
|
||||
import os
|
||||
import io
|
||||
import re
|
||||
import base64
|
||||
import logging
|
||||
import tempfile
|
||||
@@ -261,23 +262,210 @@ def process_document(image: Image.Image) -> dict:
|
||||
|
||||
|
||||
def result_to_markdown(result: dict) -> str:
|
||||
"""Convert result to Markdown format"""
|
||||
"""Convert result to Markdown format with structural hints for LLM processing.
|
||||
|
||||
Adds positional and type-based formatting to help downstream LLMs
|
||||
understand document structure:
|
||||
- Tables are marked with **[TABLE]** prefix
|
||||
- Header zone content (top 15%) is bolded
|
||||
- Footer zone content (bottom 15%) is separated with horizontal rule
|
||||
- Titles are formatted as # headers
|
||||
- Figures/charts are marked with *[Figure: ...]*
|
||||
"""
|
||||
lines = []
|
||||
image_height = result.get("image_size", [0, 1000])[1]
|
||||
|
||||
for block in result.get("blocks", []):
|
||||
block_type = block.get("type", "text")
|
||||
content = block.get("content", "")
|
||||
block_type = block.get("type", "text").lower()
|
||||
content = block.get("content", "").strip()
|
||||
bbox = block.get("bbox", [])
|
||||
|
||||
if "table" in block_type.lower():
|
||||
lines.append(f"\n{content}\n")
|
||||
elif "formula" in block_type.lower():
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Determine position zone (top 15%, middle, bottom 15%)
|
||||
y_pos = bbox[1] if bbox and len(bbox) > 1 else 0
|
||||
y_end = bbox[3] if bbox and len(bbox) > 3 else y_pos
|
||||
is_header_zone = y_pos < image_height * 0.15
|
||||
is_footer_zone = y_end > image_height * 0.85
|
||||
|
||||
# Format based on type and position
|
||||
if "table" in block_type:
|
||||
lines.append(f"\n**[TABLE]**\n{content}\n")
|
||||
elif "title" in block_type:
|
||||
lines.append(f"# {content}")
|
||||
elif "formula" in block_type or "math" in block_type:
|
||||
lines.append(f"\n$$\n{content}\n$$\n")
|
||||
elif "figure" in block_type or "chart" in block_type:
|
||||
lines.append(f"*[Figure: {content}]*")
|
||||
elif is_header_zone:
|
||||
lines.append(f"**{content}**")
|
||||
elif is_footer_zone:
|
||||
lines.append(f"---\n{content}")
|
||||
else:
|
||||
lines.append(content)
|
||||
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
def parse_markdown_table(content: str) -> str:
|
||||
"""Convert table content to HTML table.
|
||||
|
||||
Handles:
|
||||
- PaddleOCR-VL format: <fcel>cell<lcel>cell<nl> (detected by <fcel> tags)
|
||||
- Pipe-delimited tables: | Header | Header |
|
||||
- Separator rows: |---|---|
|
||||
- Returns HTML <table> structure
|
||||
"""
|
||||
content_stripped = content.strip()
|
||||
|
||||
# Check for PaddleOCR-VL table format (<fcel>, <lcel>, <ecel>, <nl>)
|
||||
if '<fcel>' in content_stripped or '<nl>' in content_stripped:
|
||||
return parse_paddleocr_table(content_stripped)
|
||||
|
||||
lines = content_stripped.split('\n')
|
||||
if not lines:
|
||||
return f'<pre>{content}</pre>'
|
||||
|
||||
# Check if it looks like a markdown table
|
||||
if not any('|' in line for line in lines):
|
||||
return f'<pre>{content}</pre>'
|
||||
|
||||
html_rows = []
|
||||
is_header = True
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('|') == False and '|' not in line:
|
||||
continue
|
||||
|
||||
# Skip separator rows (|---|---|)
|
||||
if re.match(r'^[\|\s\-:]+$', line):
|
||||
is_header = False
|
||||
continue
|
||||
|
||||
# Parse cells
|
||||
cells = [c.strip() for c in line.split('|')]
|
||||
cells = [c for c in cells if c] # Remove empty from edges
|
||||
|
||||
if is_header:
|
||||
row = '<tr>' + ''.join(f'<th>{c}</th>' for c in cells) + '</tr>'
|
||||
html_rows.append(f'<thead>{row}</thead>')
|
||||
is_header = False
|
||||
else:
|
||||
row = '<tr>' + ''.join(f'<td>{c}</td>' for c in cells) + '</tr>'
|
||||
html_rows.append(row)
|
||||
|
||||
if html_rows:
|
||||
# Wrap body rows in tbody
|
||||
header = html_rows[0] if '<thead>' in html_rows[0] else ''
|
||||
body_rows = [r for r in html_rows if '<thead>' not in r]
|
||||
body = f'<tbody>{"".join(body_rows)}</tbody>' if body_rows else ''
|
||||
return f'<table>{header}{body}</table>'
|
||||
|
||||
return f'<pre>{content}</pre>'
|
||||
|
||||
|
||||
def parse_paddleocr_table(content: str) -> str:
|
||||
"""Convert PaddleOCR-VL table format to HTML table.
|
||||
|
||||
PaddleOCR-VL uses:
|
||||
- <fcel> = first cell in a row
|
||||
- <lcel> = subsequent cells
|
||||
- <ecel> = empty cell
|
||||
- <nl> = row separator (newline)
|
||||
|
||||
Example input:
|
||||
<fcel>Header1<lcel>Header2<nl><fcel>Value1<lcel>Value2<nl>
|
||||
"""
|
||||
# Split into rows by <nl>
|
||||
rows_raw = re.split(r'<nl>', content)
|
||||
html_rows = []
|
||||
is_first_row = True
|
||||
|
||||
for row_content in rows_raw:
|
||||
row_content = row_content.strip()
|
||||
if not row_content:
|
||||
continue
|
||||
|
||||
# Extract cells: split by <fcel>, <lcel>, or <ecel>
|
||||
# Each cell is the text between these markers
|
||||
cells = []
|
||||
|
||||
# Pattern to match cell markers and capture content
|
||||
# Content is everything between markers
|
||||
parts = re.split(r'<fcel>|<lcel>|<ecel>', row_content)
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if part:
|
||||
cells.append(part)
|
||||
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
# First row is header
|
||||
if is_first_row:
|
||||
row_html = '<tr>' + ''.join(f'<th>{c}</th>' for c in cells) + '</tr>'
|
||||
html_rows.append(f'<thead>{row_html}</thead>')
|
||||
is_first_row = False
|
||||
else:
|
||||
row_html = '<tr>' + ''.join(f'<td>{c}</td>' for c in cells) + '</tr>'
|
||||
html_rows.append(row_html)
|
||||
|
||||
if html_rows:
|
||||
header = html_rows[0] if '<thead>' in html_rows[0] else ''
|
||||
body_rows = [r for r in html_rows if '<thead>' not in r]
|
||||
body = f'<tbody>{"".join(body_rows)}</tbody>' if body_rows else ''
|
||||
return f'<table>{header}{body}</table>'
|
||||
|
||||
return f'<pre>{content}</pre>'
|
||||
|
||||
|
||||
def result_to_html(result: dict) -> str:
|
||||
"""Convert result to semantic HTML for optimal LLM processing.
|
||||
|
||||
Uses semantic HTML5 tags with position metadata as data-* attributes.
|
||||
Markdown tables are converted to proper HTML <table> tags for
|
||||
unambiguous parsing by downstream LLMs.
|
||||
"""
|
||||
parts = []
|
||||
image_height = result.get("image_size", [0, 1000])[1]
|
||||
|
||||
parts.append('<!DOCTYPE html><html><body>')
|
||||
|
||||
for block in result.get("blocks", []):
|
||||
block_type = block.get("type", "text").lower()
|
||||
content = block.get("content", "").strip()
|
||||
bbox = block.get("bbox", [])
|
||||
|
||||
if not content:
|
||||
continue
|
||||
|
||||
# Position metadata
|
||||
y_pos = bbox[1] / image_height if bbox and len(bbox) > 1 else 0
|
||||
data_attrs = f'data-type="{block_type}" data-y="{y_pos:.2f}"'
|
||||
|
||||
# Format based on type
|
||||
if "table" in block_type:
|
||||
table_html = parse_markdown_table(content)
|
||||
parts.append(f'<section {data_attrs} class="table-region">{table_html}</section>')
|
||||
elif "title" in block_type:
|
||||
parts.append(f'<h1 {data_attrs}>{content}</h1>')
|
||||
elif "formula" in block_type or "math" in block_type:
|
||||
parts.append(f'<div {data_attrs} class="formula"><code>{content}</code></div>')
|
||||
elif "figure" in block_type or "chart" in block_type:
|
||||
parts.append(f'<figure {data_attrs}><figcaption>{content}</figcaption></figure>')
|
||||
elif y_pos < 0.15:
|
||||
parts.append(f'<header {data_attrs}><strong>{content}</strong></header>')
|
||||
elif y_pos > 0.85:
|
||||
parts.append(f'<footer {data_attrs}>{content}</footer>')
|
||||
else:
|
||||
parts.append(f'<p {data_attrs}>{content}</p>')
|
||||
|
||||
parts.append('</body></html>')
|
||||
return '\n'.join(parts)
|
||||
|
||||
|
||||
# Request/Response models
|
||||
class ParseRequest(BaseModel):
|
||||
image: str # base64 encoded image
|
||||
@@ -331,7 +519,7 @@ async def health_check():
|
||||
async def supported_formats():
|
||||
"""List supported output formats"""
|
||||
return {
|
||||
"output_formats": ["json", "markdown"],
|
||||
"output_formats": ["json", "markdown", "html"],
|
||||
"image_formats": ["PNG", "JPEG", "WebP", "BMP", "GIF", "TIFF"],
|
||||
"capabilities": [
|
||||
"Layout detection (PP-DocLayoutV2)",
|
||||
@@ -356,6 +544,9 @@ async def parse_document_endpoint(request: ParseRequest):
|
||||
if request.output_format == "markdown":
|
||||
markdown = result_to_markdown(result)
|
||||
output = {"markdown": markdown}
|
||||
elif request.output_format == "html":
|
||||
html = result_to_html(result)
|
||||
output = {"html": html}
|
||||
else:
|
||||
output = result
|
||||
|
||||
@@ -408,6 +599,8 @@ async def chat_completions(request: dict):
|
||||
|
||||
if output_format == "markdown":
|
||||
content = result_to_markdown(result)
|
||||
elif output_format == "html":
|
||||
content = result_to_html(result)
|
||||
else:
|
||||
content = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@host.today/ht-docker-ai",
|
||||
"version": "1.7.1",
|
||||
"version": "1.10.0",
|
||||
"type": "module",
|
||||
"private": false,
|
||||
"description": "Docker images for AI vision-language models including MiniCPM-V 4.5",
|
||||
|
||||
296
readme.md
296
readme.md
@@ -1,23 +1,40 @@
|
||||
# @host.today/ht-docker-ai
|
||||
# @host.today/ht-docker-ai 🚀
|
||||
|
||||
Docker images for AI vision-language models, starting with MiniCPM-V 4.5.
|
||||
Production-ready Docker images for state-of-the-art AI Vision-Language Models. Run powerful multimodal AI locally with GPU acceleration or CPU fallback—no cloud API keys required.
|
||||
|
||||
## Overview
|
||||
## Issue Reporting and Security
|
||||
|
||||
This project provides ready-to-use Docker containers for running state-of-the-art AI vision-language models. Built on Ollama for simplified model management and a consistent REST API.
|
||||
For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
|
||||
|
||||
## Available Images
|
||||
## 🎯 What's Included
|
||||
|
||||
| Tag | Description | Requirements |
|
||||
|-----|-------------|--------------|
|
||||
| `minicpm45v` | MiniCPM-V 4.5 with GPU support | NVIDIA GPU, 9-18GB VRAM |
|
||||
| `minicpm45v-cpu` | MiniCPM-V 4.5 CPU-only | 8GB+ RAM |
|
||||
| `latest` | Alias for `minicpm45v` | NVIDIA GPU |
|
||||
| Model | Parameters | Best For | API |
|
||||
|-------|-----------|----------|-----|
|
||||
| **MiniCPM-V 4.5** | 8B | General vision understanding, image analysis, multi-image | Ollama-compatible |
|
||||
| **PaddleOCR-VL** | 0.9B | Document parsing, table extraction, OCR | OpenAI-compatible |
|
||||
|
||||
## Quick Start
|
||||
## 📦 Available Images
|
||||
|
||||
### GPU (Recommended)
|
||||
```
|
||||
code.foss.global/host.today/ht-docker-ai:<tag>
|
||||
```
|
||||
|
||||
| Tag | Model | Hardware | Port |
|
||||
|-----|-------|----------|------|
|
||||
| `minicpm45v` / `latest` | MiniCPM-V 4.5 | NVIDIA GPU (9-18GB VRAM) | 11434 |
|
||||
| `minicpm45v-cpu` | MiniCPM-V 4.5 | CPU only (8GB+ RAM) | 11434 |
|
||||
| `paddleocr-vl` / `paddleocr-vl-gpu` | PaddleOCR-VL | NVIDIA GPU | 8000 |
|
||||
| `paddleocr-vl-cpu` | PaddleOCR-VL | CPU only | 8000 |
|
||||
|
||||
---
|
||||
|
||||
## 🖼️ MiniCPM-V 4.5
|
||||
|
||||
A GPT-4o level multimodal LLM from OpenBMB—handles image understanding, OCR, multi-image analysis, and visual reasoning across 30+ languages.
|
||||
|
||||
### Quick Start
|
||||
|
||||
**GPU (Recommended):**
|
||||
```bash
|
||||
docker run -d \
|
||||
--name minicpm \
|
||||
@@ -27,8 +44,7 @@ docker run -d \
|
||||
code.foss.global/host.today/ht-docker-ai:minicpm45v
|
||||
```
|
||||
|
||||
### CPU Only
|
||||
|
||||
**CPU Only:**
|
||||
```bash
|
||||
docker run -d \
|
||||
--name minicpm \
|
||||
@@ -37,18 +53,16 @@ docker run -d \
|
||||
code.foss.global/host.today/ht-docker-ai:minicpm45v-cpu
|
||||
```
|
||||
|
||||
## API Usage
|
||||
> 💡 **Pro tip:** Mount the volume to persist downloaded models (~5GB). Without it, models re-download on every container start.
|
||||
|
||||
The container exposes the Ollama API on port 11434.
|
||||
|
||||
### List Available Models
|
||||
### API Examples
|
||||
|
||||
**List models:**
|
||||
```bash
|
||||
curl http://localhost:11434/api/tags
|
||||
```
|
||||
|
||||
### Generate Text from Image
|
||||
|
||||
**Analyze an image:**
|
||||
```bash
|
||||
curl http://localhost:11434/api/generate -d '{
|
||||
"model": "minicpm-v",
|
||||
@@ -57,60 +71,128 @@ curl http://localhost:11434/api/generate -d '{
|
||||
}'
|
||||
```
|
||||
|
||||
### Chat with Vision
|
||||
|
||||
**Chat with vision:**
|
||||
```bash
|
||||
curl http://localhost:11434/api/chat -d '{
|
||||
"model": "minicpm-v",
|
||||
"messages": [
|
||||
{
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": "Describe this image in detail",
|
||||
"images": ["<base64-encoded-image>"]
|
||||
}
|
||||
]
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
### Hardware Requirements
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `MODEL_NAME` | `minicpm-v` | Model to pull on startup |
|
||||
| `OLLAMA_HOST` | `0.0.0.0` | Host address for API |
|
||||
| `OLLAMA_ORIGINS` | `*` | Allowed CORS origins |
|
||||
| Variant | VRAM/RAM | Notes |
|
||||
|---------|----------|-------|
|
||||
| GPU (int4 quantized) | 9GB VRAM | Recommended for most use cases |
|
||||
| GPU (full precision) | 18GB VRAM | Maximum quality |
|
||||
| CPU (GGUF) | 8GB+ RAM | Slower but accessible |
|
||||
|
||||
## Hardware Requirements
|
||||
---
|
||||
|
||||
### GPU Variant (`minicpm45v`)
|
||||
## 📄 PaddleOCR-VL
|
||||
|
||||
- NVIDIA GPU with CUDA support
|
||||
- Minimum 9GB VRAM (int4 quantized)
|
||||
- Recommended 18GB VRAM (full precision)
|
||||
- NVIDIA Container Toolkit installed
|
||||
A specialized 0.9B Vision-Language Model optimized for document parsing. Native support for tables, formulas, charts, and text extraction in 109 languages.
|
||||
|
||||
### CPU Variant (`minicpm45v-cpu`)
|
||||
### Quick Start
|
||||
|
||||
- Minimum 8GB RAM
|
||||
- Recommended 16GB+ RAM for better performance
|
||||
- No GPU required
|
||||
**GPU:**
|
||||
```bash
|
||||
docker run -d \
|
||||
--name paddleocr \
|
||||
--gpus all \
|
||||
-p 8000:8000 \
|
||||
-v hf-cache:/root/.cache/huggingface \
|
||||
code.foss.global/host.today/ht-docker-ai:paddleocr-vl
|
||||
```
|
||||
|
||||
## Model Information
|
||||
**CPU:**
|
||||
```bash
|
||||
docker run -d \
|
||||
--name paddleocr \
|
||||
-p 8000:8000 \
|
||||
-v hf-cache:/root/.cache/huggingface \
|
||||
code.foss.global/host.today/ht-docker-ai:paddleocr-vl-cpu
|
||||
```
|
||||
|
||||
**MiniCPM-V 4.5** is a GPT-4o level multimodal large language model developed by OpenBMB.
|
||||
### OpenAI-Compatible API
|
||||
|
||||
- **Parameters**: 8B (Qwen3-8B + SigLIP2-400M)
|
||||
- **Capabilities**: Image understanding, OCR, multi-image analysis
|
||||
- **Languages**: 30+ languages including English, Chinese, French, Spanish
|
||||
PaddleOCR-VL exposes a fully OpenAI-compatible `/v1/chat/completions` endpoint:
|
||||
|
||||
## Docker Compose Example
|
||||
```bash
|
||||
curl http://localhost:8000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "paddleocr-vl",
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,<base64>"}},
|
||||
{"type": "text", "text": "Table Recognition:"}
|
||||
]
|
||||
}],
|
||||
"max_tokens": 8192
|
||||
}'
|
||||
```
|
||||
|
||||
### Task Prompts
|
||||
|
||||
| Prompt | Output | Use Case |
|
||||
|--------|--------|----------|
|
||||
| `OCR:` | Plain text | General text extraction |
|
||||
| `Table Recognition:` | Markdown table | Invoices, bank statements, spreadsheets |
|
||||
| `Formula Recognition:` | LaTeX | Math equations, scientific notation |
|
||||
| `Chart Recognition:` | Description | Graphs and visualizations |
|
||||
|
||||
### API Endpoints
|
||||
|
||||
| Endpoint | Method | Description |
|
||||
|----------|--------|-------------|
|
||||
| `/health` | GET | Health check with model/device info |
|
||||
| `/formats` | GET | Supported image formats and input methods |
|
||||
| `/v1/models` | GET | List available models |
|
||||
| `/v1/chat/completions` | POST | OpenAI-compatible chat completions |
|
||||
| `/ocr` | POST | Legacy OCR endpoint |
|
||||
|
||||
### Image Input Methods
|
||||
|
||||
PaddleOCR-VL accepts images in multiple formats:
|
||||
|
||||
```javascript
|
||||
// Base64 data URL
|
||||
"data:image/png;base64,iVBORw0KGgo..."
|
||||
|
||||
// HTTP URL
|
||||
"https://example.com/document.png"
|
||||
|
||||
// Raw base64
|
||||
"iVBORw0KGgo..."
|
||||
```
|
||||
|
||||
**Supported formats:** PNG, JPEG, WebP, BMP, GIF, TIFF
|
||||
|
||||
**Optimal resolution:** 1080p–2K. Images are automatically scaled for best results.
|
||||
|
||||
### Performance
|
||||
|
||||
| Mode | Speed per Page |
|
||||
|------|----------------|
|
||||
| GPU (CUDA) | 2–5 seconds |
|
||||
| CPU | 30–60 seconds |
|
||||
|
||||
---
|
||||
|
||||
## 🐳 Docker Compose
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
services:
|
||||
# General vision tasks
|
||||
minicpm:
|
||||
image: code.foss.global/host.today/ht-docker-ai:minicpm45v
|
||||
container_name: minicpm
|
||||
ports:
|
||||
- "11434:11434"
|
||||
volumes:
|
||||
@@ -124,11 +206,50 @@ services:
|
||||
capabilities: [gpu]
|
||||
restart: unless-stopped
|
||||
|
||||
# Document parsing / OCR
|
||||
paddleocr:
|
||||
image: code.foss.global/host.today/ht-docker-ai:paddleocr-vl
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- hf-cache:/root/.cache/huggingface
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
ollama-data:
|
||||
hf-cache:
|
||||
```
|
||||
|
||||
## Building Locally
|
||||
---
|
||||
|
||||
## ⚙️ Environment Variables
|
||||
|
||||
### MiniCPM-V 4.5
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `MODEL_NAME` | `minicpm-v` | Ollama model to pull on startup |
|
||||
| `OLLAMA_HOST` | `0.0.0.0` | API bind address |
|
||||
| `OLLAMA_ORIGINS` | `*` | Allowed CORS origins |
|
||||
|
||||
### PaddleOCR-VL
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `MODEL_NAME` | `PaddlePaddle/PaddleOCR-VL` | HuggingFace model ID |
|
||||
| `SERVER_HOST` | `0.0.0.0` | API bind address |
|
||||
| `SERVER_PORT` | `8000` | API port |
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Building from Source
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
@@ -142,6 +263,77 @@ cd ht-docker-ai
|
||||
./test-images.sh
|
||||
```
|
||||
|
||||
## License
|
||||
---
|
||||
|
||||
MIT - Task Venture Capital GmbH
|
||||
## 🏗️ Architecture Notes
|
||||
|
||||
### Dual-VLM Consensus Strategy
|
||||
|
||||
For production document extraction, consider using both models together:
|
||||
|
||||
1. **Pass 1:** MiniCPM-V visual extraction (images → JSON)
|
||||
2. **Pass 2:** PaddleOCR-VL table recognition (images → markdown → JSON)
|
||||
3. **Consensus:** If results match → Done (fast path)
|
||||
4. **Pass 3+:** Additional visual passes if needed
|
||||
|
||||
This dual-VLM approach catches extraction errors that single models miss.
|
||||
|
||||
### Why This Works
|
||||
|
||||
- **Different architectures:** Two independent models cross-validate each other
|
||||
- **Specialized strengths:** PaddleOCR-VL excels at tables; MiniCPM-V handles general vision
|
||||
- **Native processing:** Both VLMs see original images—no intermediate HTML/structure loss
|
||||
|
||||
---
|
||||
|
||||
## 🔍 Troubleshooting
|
||||
|
||||
### Model download hangs
|
||||
```bash
|
||||
docker logs -f <container-name>
|
||||
```
|
||||
Model downloads can take several minutes (~5GB for MiniCPM-V).
|
||||
|
||||
### Out of memory
|
||||
- **GPU:** Use the CPU variant or upgrade VRAM
|
||||
- **CPU:** Increase container memory: `--memory=16g`
|
||||
|
||||
### API not responding
|
||||
1. Check container health: `docker ps`
|
||||
2. Review logs: `docker logs <container>`
|
||||
3. Verify port: `curl localhost:11434/api/tags` or `curl localhost:8000/health`
|
||||
|
||||
### Enable NVIDIA GPU support on host
|
||||
```bash
|
||||
# Install NVIDIA Container Toolkit
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
|
||||
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
|
||||
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
|
||||
sudo nvidia-ctk runtime configure --runtime=docker
|
||||
sudo systemctl restart docker
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## License and Legal Information
|
||||
|
||||
This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
|
||||
|
||||
**Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
|
||||
|
||||
### Trademarks
|
||||
|
||||
This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
|
||||
|
||||
Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
|
||||
|
||||
### Company Information
|
||||
|
||||
Task Venture Capital GmbH
|
||||
Registered at District Court Bremen HRB 35230 HB, Germany
|
||||
|
||||
For any legal inquiries or further information, please contact us via email at hello@task.vc.
|
||||
|
||||
By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
|
||||
|
||||
@@ -311,9 +311,8 @@ export async function ensureOllamaModel(modelName: string): Promise<boolean> {
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
const models = data.models || [];
|
||||
const exists = models.some((m: { name: string }) =>
|
||||
m.name === modelName || m.name.startsWith(modelName.split(':')[0])
|
||||
);
|
||||
// Exact match required - don't match on prefix
|
||||
const exists = models.some((m: { name: string }) => m.name === modelName);
|
||||
|
||||
if (exists) {
|
||||
console.log(`[Ollama] Model already available: ${modelName}`);
|
||||
@@ -358,3 +357,29 @@ export async function ensureQwen25(): Promise<boolean> {
|
||||
// Then ensure the Qwen2.5 model is pulled
|
||||
return ensureOllamaModel('qwen2.5:7b');
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure Ministral 3 8B model is available (for structured JSON extraction)
|
||||
* Ministral 3 has native JSON output support and OCR-style document extraction
|
||||
*/
|
||||
export async function ensureMinistral3(): Promise<boolean> {
|
||||
// First ensure the Ollama service (MiniCPM container) is running
|
||||
const ollamaOk = await ensureMiniCpm();
|
||||
if (!ollamaOk) return false;
|
||||
|
||||
// Then ensure the Ministral 3 8B model is pulled
|
||||
return ensureOllamaModel('ministral-3:8b');
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure Qwen3-VL 8B model is available (vision-language model)
|
||||
* Q4_K_M quantization (~5GB) - fits in 15GB VRAM with room to spare
|
||||
*/
|
||||
export async function ensureQwen3Vl(): Promise<boolean> {
|
||||
// First ensure the Ollama service is running
|
||||
const ollamaOk = await ensureMiniCpm();
|
||||
if (!ollamaOk) return false;
|
||||
|
||||
// Then ensure Qwen3-VL 8B is pulled
|
||||
return ensureOllamaModel('qwen3-vl:8b');
|
||||
}
|
||||
|
||||
348
test/test.bankstatements.ministral3.ts
Normal file
348
test/test.bankstatements.ministral3.ts
Normal file
@@ -0,0 +1,348 @@
|
||||
/**
|
||||
* Bank Statement extraction using Ministral 3 Vision (Direct)
|
||||
*
|
||||
* NO OCR pipeline needed - Ministral 3 has built-in vision encoder:
|
||||
* 1. Convert PDF to images
|
||||
* 2. Send images directly to Ministral 3 via Ollama
|
||||
* 3. Extract transactions as structured JSON
|
||||
*/
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import * as os from 'os';
|
||||
import { ensureMinistral3 } from './helpers/docker.js';
|
||||
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const VISION_MODEL = 'ministral-3:8b';
|
||||
|
||||
interface ITransaction {
|
||||
date: string;
|
||||
counterparty: string;
|
||||
amount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert PDF to PNG images using ImageMagick
|
||||
*/
|
||||
function convertPdfToImages(pdfPath: string): string[] {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
|
||||
const outputPattern = path.join(tempDir, 'page-%d.png');
|
||||
|
||||
try {
|
||||
execSync(
|
||||
`convert -density 200 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
|
||||
{ stdio: 'pipe' }
|
||||
);
|
||||
|
||||
const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
|
||||
const images: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const imagePath = path.join(tempDir, file);
|
||||
const imageData = fs.readFileSync(imagePath);
|
||||
images.push(imageData.toString('base64'));
|
||||
}
|
||||
|
||||
return images;
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract transactions from a single page image using Ministral 3 Vision
|
||||
*/
|
||||
async function extractTransactionsFromPage(image: string, pageNum: number): Promise<ITransaction[]> {
|
||||
console.log(` [Vision] Processing page ${pageNum}`);
|
||||
|
||||
// JSON schema for array of transactions
|
||||
const transactionSchema = {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
date: { type: 'string', description: 'Transaction date in YYYY-MM-DD format' },
|
||||
counterparty: { type: 'string', description: 'Name of the other party' },
|
||||
amount: { type: 'number', description: 'Amount (negative for debits, positive for credits)' },
|
||||
},
|
||||
required: ['date', 'counterparty', 'amount'],
|
||||
},
|
||||
};
|
||||
|
||||
const prompt = `Extract ALL bank transactions from this bank statement page.
|
||||
|
||||
For each transaction, extract:
|
||||
- date: Transaction date in YYYY-MM-DD format
|
||||
- counterparty: The name/description of the other party (merchant, payee, etc.)
|
||||
- amount: The amount as a number (NEGATIVE for debits/expenses, POSITIVE for credits/income)
|
||||
|
||||
Return a JSON array of transactions. If no transactions visible, return empty array [].
|
||||
Example: [{"date":"2021-06-01","counterparty":"AMAZON","amount":-50.00}]`;
|
||||
|
||||
const response = await fetch(`${OLLAMA_URL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: VISION_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
images: [image],
|
||||
},
|
||||
],
|
||||
format: transactionSchema,
|
||||
stream: true,
|
||||
options: {
|
||||
num_predict: 4096, // Bank statements can have many transactions
|
||||
temperature: 0.0,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error('No response body');
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
let fullText = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
const lines = chunk.split('\n').filter((l) => l.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const json = JSON.parse(line);
|
||||
if (json.message?.content) {
|
||||
fullText += json.message.content;
|
||||
}
|
||||
} catch {
|
||||
// Skip invalid JSON lines
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
let jsonStr = fullText.trim();
|
||||
|
||||
if (jsonStr.startsWith('```json')) jsonStr = jsonStr.slice(7);
|
||||
else if (jsonStr.startsWith('```')) jsonStr = jsonStr.slice(3);
|
||||
if (jsonStr.endsWith('```')) jsonStr = jsonStr.slice(0, -3);
|
||||
jsonStr = jsonStr.trim();
|
||||
|
||||
// Find array boundaries
|
||||
const startIdx = jsonStr.indexOf('[');
|
||||
const endIdx = jsonStr.lastIndexOf(']') + 1;
|
||||
|
||||
if (startIdx < 0 || endIdx <= startIdx) {
|
||||
console.log(` [Page ${pageNum}] No transactions found`);
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(jsonStr.substring(startIdx, endIdx));
|
||||
console.log(` [Page ${pageNum}] Found ${parsed.length} transactions`);
|
||||
return parsed.map((t: { date?: string; counterparty?: string; amount?: number }) => ({
|
||||
date: t.date || '',
|
||||
counterparty: t.counterparty || '',
|
||||
amount: parseFloat(String(t.amount)) || 0,
|
||||
}));
|
||||
} catch (e) {
|
||||
console.log(` [Page ${pageNum}] Parse error: ${e}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all transactions from all pages
|
||||
*/
|
||||
async function extractAllTransactions(images: string[]): Promise<ITransaction[]> {
|
||||
const allTransactions: ITransaction[] = [];
|
||||
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
const pageTransactions = await extractTransactionsFromPage(images[i], i + 1);
|
||||
allTransactions.push(...pageTransactions);
|
||||
}
|
||||
|
||||
return allTransactions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize date to YYYY-MM-DD
|
||||
*/
|
||||
function normalizeDate(dateStr: string): string {
|
||||
if (!dateStr) return '';
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
|
||||
|
||||
// Handle DD/MM/YYYY or DD.MM.YYYY
|
||||
const match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
|
||||
if (match) {
|
||||
return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
return dateStr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare extracted transactions vs expected
|
||||
*/
|
||||
function compareTransactions(
|
||||
extracted: ITransaction[],
|
||||
expected: ITransaction[]
|
||||
): { matchRate: number; matched: number; missed: number; extra: number; errors: string[] } {
|
||||
const errors: string[] = [];
|
||||
let matched = 0;
|
||||
|
||||
// Normalize all dates
|
||||
const normalizedExtracted = extracted.map((t) => ({
|
||||
...t,
|
||||
date: normalizeDate(t.date),
|
||||
counterparty: t.counterparty.toUpperCase().trim(),
|
||||
}));
|
||||
|
||||
const normalizedExpected = expected.map((t) => ({
|
||||
...t,
|
||||
date: normalizeDate(t.date),
|
||||
counterparty: t.counterparty.toUpperCase().trim(),
|
||||
}));
|
||||
|
||||
// Try to match each expected transaction
|
||||
const matchedIndices = new Set<number>();
|
||||
|
||||
for (const exp of normalizedExpected) {
|
||||
let found = false;
|
||||
|
||||
for (let i = 0; i < normalizedExtracted.length; i++) {
|
||||
if (matchedIndices.has(i)) continue;
|
||||
|
||||
const ext = normalizedExtracted[i];
|
||||
|
||||
// Match by date + amount (counterparty names can vary)
|
||||
if (ext.date === exp.date && Math.abs(ext.amount - exp.amount) < 0.02) {
|
||||
matched++;
|
||||
matchedIndices.add(i);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
errors.push(`Missing: ${exp.date} | ${exp.counterparty} | ${exp.amount}`);
|
||||
}
|
||||
}
|
||||
|
||||
const missed = expected.length - matched;
|
||||
const extra = extracted.length - matched;
|
||||
const matchRate = expected.length > 0 ? (matched / expected.length) * 100 : 0;
|
||||
|
||||
return { matchRate, matched, missed, extra, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find test cases (PDF + JSON pairs in .nogit/)
|
||||
*/
|
||||
function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
|
||||
const testDir = path.join(process.cwd(), '.nogit');
|
||||
if (!fs.existsSync(testDir)) return [];
|
||||
|
||||
const files = fs.readdirSync(testDir);
|
||||
const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
|
||||
|
||||
for (const pdf of files.filter((f) => f.endsWith('.pdf'))) {
|
||||
const baseName = pdf.replace('.pdf', '');
|
||||
const jsonFile = `${baseName}.json`;
|
||||
if (files.includes(jsonFile)) {
|
||||
// Skip invoice files - only bank statements
|
||||
if (!baseName.includes('invoice')) {
|
||||
testCases.push({
|
||||
name: baseName,
|
||||
pdfPath: path.join(testDir, pdf),
|
||||
jsonPath: path.join(testDir, jsonFile),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return testCases.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
// Tests
|
||||
|
||||
tap.test('setup: ensure Ministral 3 is running', async () => {
|
||||
console.log('\n[Setup] Checking Ministral 3...\n');
|
||||
const ok = await ensureMinistral3();
|
||||
expect(ok).toBeTrue();
|
||||
console.log('\n[Setup] Ready!\n');
|
||||
});
|
||||
|
||||
const testCases = findTestCases();
|
||||
console.log(`\nFound ${testCases.length} bank statement test cases (Ministral 3 Vision)\n`);
|
||||
|
||||
let totalMatched = 0;
|
||||
let totalExpected = 0;
|
||||
const times: number[] = [];
|
||||
|
||||
for (const testCase of testCases) {
|
||||
tap.test(`should extract bank statement: ${testCase.name}`, async () => {
|
||||
const expected: ITransaction[] = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
|
||||
console.log(`\n=== ${testCase.name} ===`);
|
||||
console.log(`Expected: ${expected.length} transactions`);
|
||||
|
||||
const start = Date.now();
|
||||
const images = convertPdfToImages(testCase.pdfPath);
|
||||
console.log(` Pages: ${images.length}`);
|
||||
|
||||
const extracted = await extractAllTransactions(images);
|
||||
const elapsed = Date.now() - start;
|
||||
times.push(elapsed);
|
||||
|
||||
console.log(` Extracted: ${extracted.length} transactions`);
|
||||
|
||||
const result = compareTransactions(extracted, expected);
|
||||
totalMatched += result.matched;
|
||||
totalExpected += expected.length;
|
||||
|
||||
console.log(` Match rate: ${result.matchRate.toFixed(1)}% (${result.matched}/${expected.length})`);
|
||||
console.log(` Missed: ${result.missed}, Extra: ${result.extra}`);
|
||||
console.log(` Time: ${(elapsed / 1000).toFixed(1)}s`);
|
||||
|
||||
if (result.errors.length > 0 && result.errors.length <= 5) {
|
||||
result.errors.forEach((e) => console.log(` - ${e}`));
|
||||
} else if (result.errors.length > 5) {
|
||||
console.log(` (${result.errors.length} missing transactions)`);
|
||||
}
|
||||
|
||||
// Consider it a pass if we match at least 70% of transactions
|
||||
expect(result.matchRate).toBeGreaterThan(70);
|
||||
});
|
||||
}
|
||||
|
||||
tap.test('summary', async () => {
|
||||
const overallMatchRate = totalExpected > 0 ? (totalMatched / totalExpected) * 100 : 0;
|
||||
const totalTime = times.reduce((a, b) => a + b, 0) / 1000;
|
||||
const avgTime = times.length > 0 ? totalTime / times.length : 0;
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` Bank Statement Extraction Summary (Ministral 3)`);
|
||||
console.log(`======================================================`);
|
||||
console.log(` Method: Ministral 3 8B Vision (Direct)`);
|
||||
console.log(` Statements: ${testCases.length}`);
|
||||
console.log(` Matched: ${totalMatched}/${totalExpected} transactions`);
|
||||
console.log(` Match rate: ${overallMatchRate.toFixed(1)}%`);
|
||||
console.log(`------------------------------------------------------`);
|
||||
console.log(` Total time: ${totalTime.toFixed(1)}s`);
|
||||
console.log(` Avg per stmt: ${avgTime.toFixed(1)}s`);
|
||||
console.log(`======================================================\n`);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
334
test/test.invoices.ministral3.ts
Normal file
334
test/test.invoices.ministral3.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* Invoice extraction using Ministral 3 Vision (Direct)
|
||||
*
|
||||
* NO PaddleOCR needed - Ministral 3 has built-in vision encoder:
|
||||
* 1. Convert PDF to images
|
||||
* 2. Send images directly to Ministral 3 via Ollama
|
||||
* 3. Extract structured JSON with native schema support
|
||||
*
|
||||
* This is the simplest possible pipeline.
|
||||
*/
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import * as os from 'os';
|
||||
import { ensureMinistral3 } from './helpers/docker.js';
|
||||
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const VISION_MODEL = 'ministral-3:8b';
|
||||
|
||||
interface IInvoice {
|
||||
invoice_number: string;
|
||||
invoice_date: string;
|
||||
vendor_name: string;
|
||||
currency: string;
|
||||
net_amount: number;
|
||||
vat_amount: number;
|
||||
total_amount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert PDF to PNG images using ImageMagick
|
||||
*/
|
||||
function convertPdfToImages(pdfPath: string): string[] {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
|
||||
const outputPattern = path.join(tempDir, 'page-%d.png');
|
||||
|
||||
try {
|
||||
// High quality conversion: 300 DPI, max quality, sharpen for better OCR
|
||||
execSync(
|
||||
`convert -density 300 -quality 100 "${pdfPath}" -background white -alpha remove -sharpen 0x1 "${outputPattern}"`,
|
||||
{ stdio: 'pipe' }
|
||||
);
|
||||
|
||||
const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
|
||||
const images: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const imagePath = path.join(tempDir, file);
|
||||
const imageData = fs.readFileSync(imagePath);
|
||||
images.push(imageData.toString('base64'));
|
||||
}
|
||||
|
||||
return images;
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract invoice data directly from images using Ministral 3 Vision
|
||||
*/
|
||||
async function extractInvoiceFromImages(images: string[]): Promise<IInvoice> {
|
||||
console.log(` [Vision] Processing ${images.length} page(s) with Ministral 3`);
|
||||
|
||||
// JSON schema for structured output
|
||||
const invoiceSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
invoice_number: { type: 'string' },
|
||||
invoice_date: { type: 'string' },
|
||||
vendor_name: { type: 'string' },
|
||||
currency: { type: 'string' },
|
||||
net_amount: { type: 'number' },
|
||||
vat_amount: { type: 'number' },
|
||||
total_amount: { type: 'number' },
|
||||
},
|
||||
required: ['invoice_number', 'invoice_date', 'vendor_name', 'currency', 'net_amount', 'vat_amount', 'total_amount'],
|
||||
};
|
||||
|
||||
const prompt = `You are an expert invoice data extraction system. Carefully analyze this invoice document and extract the following fields with high precision.
|
||||
|
||||
INVOICE NUMBER:
|
||||
- Look for labels: "Invoice No", "Invoice #", "Invoice Number", "Rechnung Nr", "Rechnungsnummer", "Document No", "Bill No", "Reference"
|
||||
- Usually alphanumeric, often starts with letters (e.g., R0014359508, INV-2024-001)
|
||||
- Located near the top of the invoice
|
||||
|
||||
INVOICE DATE:
|
||||
- Look for labels: "Invoice Date", "Date", "Datum", "Rechnungsdatum", "Issue Date", "Bill Date"
|
||||
- Convert ANY date format to YYYY-MM-DD (e.g., 14/10/2021 → 2021-10-14, Oct 14, 2021 → 2021-10-14)
|
||||
- Usually near the invoice number
|
||||
|
||||
VENDOR NAME:
|
||||
- The company ISSUING the invoice (not the recipient)
|
||||
- Found in letterhead, logo area, or header - typically the largest/most prominent company name
|
||||
- Examples: "Hetzner Online GmbH", "Adobe Inc", "DigitalOcean LLC"
|
||||
|
||||
CURRENCY:
|
||||
- Detect from symbols: € = EUR, $ = USD, £ = GBP
|
||||
- Or from text: "EUR", "USD", "GBP"
|
||||
- Default to EUR if unclear
|
||||
|
||||
AMOUNTS (Critical - read carefully!):
|
||||
- total_amount: The FINAL amount due/payable - look for "Total", "Grand Total", "Amount Due", "Balance Due", "Gesamtbetrag", "Endbetrag"
|
||||
- net_amount: Subtotal BEFORE tax - look for "Subtotal", "Net", "Netto", "excl. VAT"
|
||||
- vat_amount: Tax amount - look for "VAT", "Tax", "MwSt", "USt", "19%", "20%"
|
||||
- For multi-page invoices: the FINAL totals are usually on the LAST page
|
||||
|
||||
Return ONLY valid JSON with the extracted values.`;
|
||||
|
||||
const response = await fetch(`${OLLAMA_URL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: VISION_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
images: images, // Send all page images
|
||||
},
|
||||
],
|
||||
format: invoiceSchema,
|
||||
stream: true,
|
||||
options: {
|
||||
num_predict: 1024,
|
||||
temperature: 0.0,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error('No response body');
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
let fullText = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
const lines = chunk.split('\n').filter((l) => l.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const json = JSON.parse(line);
|
||||
if (json.message?.content) {
|
||||
fullText += json.message.content;
|
||||
}
|
||||
} catch {
|
||||
// Skip invalid JSON lines
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
let jsonStr = fullText.trim();
|
||||
|
||||
if (jsonStr.startsWith('```json')) jsonStr = jsonStr.slice(7);
|
||||
else if (jsonStr.startsWith('```')) jsonStr = jsonStr.slice(3);
|
||||
if (jsonStr.endsWith('```')) jsonStr = jsonStr.slice(0, -3);
|
||||
jsonStr = jsonStr.trim();
|
||||
|
||||
const startIdx = jsonStr.indexOf('{');
|
||||
const endIdx = jsonStr.lastIndexOf('}') + 1;
|
||||
|
||||
if (startIdx < 0 || endIdx <= startIdx) {
|
||||
throw new Error(`No JSON found: ${fullText.substring(0, 200)}`);
|
||||
}
|
||||
|
||||
const parsed = JSON.parse(jsonStr.substring(startIdx, endIdx));
|
||||
|
||||
return {
|
||||
invoice_number: parsed.invoice_number || null,
|
||||
invoice_date: parsed.invoice_date || null,
|
||||
vendor_name: parsed.vendor_name || null,
|
||||
currency: parsed.currency || 'EUR',
|
||||
net_amount: parseFloat(parsed.net_amount) || 0,
|
||||
vat_amount: parseFloat(parsed.vat_amount) || 0,
|
||||
total_amount: parseFloat(parsed.total_amount) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Normalize date to YYYY-MM-DD
|
||||
*/
|
||||
function normalizeDate(dateStr: string | null): string {
|
||||
if (!dateStr) return '';
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
|
||||
|
||||
const monthMap: Record<string, string> = {
|
||||
JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
|
||||
JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
|
||||
};
|
||||
|
||||
let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
|
||||
if (match) {
|
||||
return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
|
||||
if (match) {
|
||||
return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
return dateStr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare extracted vs expected
|
||||
*/
|
||||
function compareInvoice(extracted: IInvoice, expected: IInvoice): { match: boolean; errors: string[] } {
|
||||
const errors: string[] = [];
|
||||
|
||||
const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
|
||||
const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
|
||||
if (extNum !== expNum) {
|
||||
errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
|
||||
}
|
||||
|
||||
if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
|
||||
errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
|
||||
}
|
||||
|
||||
if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
|
||||
errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
|
||||
}
|
||||
|
||||
if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
|
||||
errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
|
||||
}
|
||||
|
||||
return { match: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find test cases
|
||||
*/
|
||||
function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
|
||||
const testDir = path.join(process.cwd(), '.nogit/invoices');
|
||||
if (!fs.existsSync(testDir)) return [];
|
||||
|
||||
const files = fs.readdirSync(testDir);
|
||||
const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
|
||||
|
||||
for (const pdf of files.filter((f) => f.endsWith('.pdf'))) {
|
||||
const baseName = pdf.replace('.pdf', '');
|
||||
const jsonFile = `${baseName}.json`;
|
||||
if (files.includes(jsonFile)) {
|
||||
testCases.push({
|
||||
name: baseName,
|
||||
pdfPath: path.join(testDir, pdf),
|
||||
jsonPath: path.join(testDir, jsonFile),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return testCases.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
// Tests
|
||||
|
||||
tap.test('setup: ensure Ministral 3 is running', async () => {
|
||||
console.log('\n[Setup] Checking Ministral 3...\n');
|
||||
const ok = await ensureMinistral3();
|
||||
expect(ok).toBeTrue();
|
||||
console.log('\n[Setup] Ready!\n');
|
||||
});
|
||||
|
||||
const testCases = findTestCases();
|
||||
console.log(`\nFound ${testCases.length} invoice test cases (Ministral 3 Vision Direct)\n`);
|
||||
|
||||
let passedCount = 0;
|
||||
let failedCount = 0;
|
||||
const times: number[] = [];
|
||||
|
||||
for (const testCase of testCases) {
|
||||
tap.test(`should extract invoice: ${testCase.name}`, async () => {
|
||||
const expected: IInvoice = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
|
||||
console.log(`\n=== ${testCase.name} ===`);
|
||||
console.log(`Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
|
||||
|
||||
const start = Date.now();
|
||||
const images = convertPdfToImages(testCase.pdfPath);
|
||||
console.log(` Pages: ${images.length}`);
|
||||
|
||||
const extracted = await extractInvoiceFromImages(images);
|
||||
console.log(` Extracted: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
|
||||
const elapsed = Date.now() - start;
|
||||
times.push(elapsed);
|
||||
|
||||
const result = compareInvoice(extracted, expected);
|
||||
|
||||
if (result.match) {
|
||||
passedCount++;
|
||||
console.log(` Result: MATCH (${(elapsed / 1000).toFixed(1)}s)`);
|
||||
} else {
|
||||
failedCount++;
|
||||
console.log(` Result: MISMATCH (${(elapsed / 1000).toFixed(1)}s)`);
|
||||
result.errors.forEach((e) => console.log(` - ${e}`));
|
||||
}
|
||||
|
||||
expect(result.match).toBeTrue();
|
||||
});
|
||||
}
|
||||
|
||||
tap.test('summary', async () => {
|
||||
const total = testCases.length;
|
||||
const accuracy = total > 0 ? (passedCount / total) * 100 : 0;
|
||||
const totalTime = times.reduce((a, b) => a + b, 0) / 1000;
|
||||
const avgTime = times.length > 0 ? totalTime / times.length : 0;
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` Invoice Extraction Summary (Ministral 3 Vision)`);
|
||||
console.log(`======================================================`);
|
||||
console.log(` Method: Ministral 3 8B Vision (Direct)`);
|
||||
console.log(` Passed: ${passedCount}/${total}`);
|
||||
console.log(` Failed: ${failedCount}/${total}`);
|
||||
console.log(` Accuracy: ${accuracy.toFixed(1)}%`);
|
||||
console.log(`------------------------------------------------------`);
|
||||
console.log(` Total time: ${totalTime.toFixed(1)}s`);
|
||||
console.log(` Avg per inv: ${avgTime.toFixed(1)}s`);
|
||||
console.log(`======================================================\n`);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
@@ -4,11 +4,13 @@
|
||||
* This tests the complete PaddleOCR-VL pipeline:
|
||||
* 1. PP-DocLayoutV2 for layout detection
|
||||
* 2. PaddleOCR-VL for recognition
|
||||
* 3. Structured Markdown output
|
||||
* 4. MiniCPM extracts invoice fields from structured Markdown
|
||||
* 3. Structured HTML output (semantic tags with proper tables)
|
||||
* 4. Qwen2.5 extracts invoice fields from structured HTML
|
||||
*
|
||||
* The structured Markdown has proper tables and formatting,
|
||||
* making it much easier for MiniCPM to extract invoice data.
|
||||
* HTML output is used instead of Markdown because:
|
||||
* - <table> tags are unambiguous (no parser variations)
|
||||
* - LLMs are heavily trained on web/HTML data
|
||||
* - Semantic tags (header, footer, section) provide clear structure
|
||||
*/
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as fs from 'fs';
|
||||
@@ -61,7 +63,7 @@ function convertPdfToImages(pdfPath: string): string[] {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse document using PaddleOCR-VL Full Pipeline (returns structured Markdown)
|
||||
* Parse document using PaddleOCR-VL Full Pipeline (returns structured HTML)
|
||||
*/
|
||||
async function parseDocument(imageBase64: string): Promise<string> {
|
||||
const response = await fetch(`${PADDLEOCR_VL_URL}/parse`, {
|
||||
@@ -69,7 +71,7 @@ async function parseDocument(imageBase64: string): Promise<string> {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
image: imageBase64,
|
||||
output_format: 'markdown',
|
||||
output_format: 'html',
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -84,57 +86,63 @@ async function parseDocument(imageBase64: string): Promise<string> {
|
||||
throw new Error(`PaddleOCR-VL error: ${data.error}`);
|
||||
}
|
||||
|
||||
return data.result?.markdown || '';
|
||||
return data.result?.html || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract invoice fields from structured Markdown using Qwen2.5 (text-only model)
|
||||
* Extract invoice fields using simple direct prompt
|
||||
* The OCR output has clearly labeled fields - just ask the LLM to read them
|
||||
*/
|
||||
async function extractInvoiceFromMarkdown(markdown: string): Promise<IInvoice> {
|
||||
// Truncate if too long
|
||||
const truncated = markdown.length > 12000 ? markdown.slice(0, 12000) : markdown;
|
||||
console.log(` [Extract] Processing ${truncated.length} chars of Markdown`);
|
||||
async function extractInvoiceFromHtml(html: string): Promise<IInvoice> {
|
||||
// OCR output is already good - just truncate if too long
|
||||
const truncated = html.length > 32000 ? html.slice(0, 32000) : html;
|
||||
console.log(` [Extract] ${truncated.length} chars of HTML`);
|
||||
|
||||
const prompt = `You are an invoice data extractor. Extract the following fields from this OCR text and return ONLY a valid JSON object.
|
||||
|
||||
Required fields:
|
||||
- invoice_number: The invoice/receipt/document number
|
||||
- invoice_date: Date in YYYY-MM-DD format (convert from any format)
|
||||
- vendor_name: Company that issued the invoice
|
||||
- currency: EUR, USD, GBP, etc.
|
||||
- net_amount: Amount before tax (number)
|
||||
- vat_amount: Tax/VAT amount (number, use 0 if reverse charge or not shown)
|
||||
- total_amount: Final total amount (number)
|
||||
|
||||
Example output format:
|
||||
{"invoice_number":"INV-123","invoice_date":"2022-01-28","vendor_name":"Adobe","currency":"EUR","net_amount":24.99,"vat_amount":0,"total_amount":24.99}
|
||||
|
||||
Rules:
|
||||
- Return ONLY the JSON object, no explanation or markdown
|
||||
- Use null for missing string fields
|
||||
- Use 0 for missing numeric fields
|
||||
- Convert dates to YYYY-MM-DD format (e.g., "28-JAN-2022" becomes "2022-01-28")
|
||||
- Extract numbers without currency symbols
|
||||
|
||||
OCR Text:
|
||||
${truncated}
|
||||
|
||||
JSON:`;
|
||||
|
||||
const payload = {
|
||||
model: TEXT_MODEL,
|
||||
prompt,
|
||||
stream: true,
|
||||
options: {
|
||||
num_predict: 512,
|
||||
temperature: 0.1,
|
||||
// JSON schema for structured output
|
||||
const invoiceSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
invoice_number: { type: 'string' },
|
||||
invoice_date: { type: 'string' },
|
||||
vendor_name: { type: 'string' },
|
||||
currency: { type: 'string' },
|
||||
net_amount: { type: 'number' },
|
||||
vat_amount: { type: 'number' },
|
||||
total_amount: { type: 'number' },
|
||||
},
|
||||
required: ['invoice_number', 'invoice_date', 'vendor_name', 'currency', 'net_amount', 'vat_amount', 'total_amount'],
|
||||
};
|
||||
|
||||
const response = await fetch(`${OLLAMA_URL}/api/generate`, {
|
||||
// Simple, direct prompt - the OCR output already has labeled fields
|
||||
const systemPrompt = `You read invoice HTML and extract labeled fields. Return JSON only.`;
|
||||
|
||||
const userPrompt = `Extract from this invoice HTML:
|
||||
- invoice_number: Find "Invoice no.", "Invoice #", "Invoice", "Rechnung", "Document No" and extract the value
|
||||
- invoice_date: Find "Invoice date", "Date", "Datum" and convert to YYYY-MM-DD format
|
||||
- vendor_name: The company name issuing the invoice (in header/letterhead)
|
||||
- currency: EUR, USD, or GBP (look for € $ £ symbols or text)
|
||||
- total_amount: Find "Total", "Grand Total", "Amount Due", "Gesamtbetrag" - the FINAL total amount
|
||||
- net_amount: Amount before VAT/tax (Subtotal, Net)
|
||||
- vat_amount: VAT/tax amount
|
||||
|
||||
HTML:
|
||||
${truncated}
|
||||
|
||||
Return ONLY valid JSON: {"invoice_number":"...", "invoice_date":"YYYY-MM-DD", "vendor_name":"...", "currency":"EUR", "net_amount":0, "vat_amount":0, "total_amount":0}`;
|
||||
|
||||
const response = await fetch(`${OLLAMA_URL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
body: JSON.stringify({
|
||||
model: TEXT_MODEL,
|
||||
messages: [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
format: invoiceSchema,
|
||||
stream: true,
|
||||
options: { num_predict: 512, temperature: 0.0 },
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -159,7 +167,9 @@ JSON:`;
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const json = JSON.parse(line);
|
||||
if (json.response) {
|
||||
if (json.message?.content) {
|
||||
fullText += json.message.content;
|
||||
} else if (json.response) {
|
||||
fullText += json.response;
|
||||
}
|
||||
} catch {
|
||||
@@ -169,17 +179,37 @@ JSON:`;
|
||||
}
|
||||
|
||||
// Extract JSON from response
|
||||
const startIdx = fullText.indexOf('{');
|
||||
const endIdx = fullText.lastIndexOf('}') + 1;
|
||||
let jsonStr = fullText.trim();
|
||||
|
||||
// Remove markdown code block if present
|
||||
if (jsonStr.startsWith('```json')) {
|
||||
jsonStr = jsonStr.slice(7);
|
||||
} else if (jsonStr.startsWith('```')) {
|
||||
jsonStr = jsonStr.slice(3);
|
||||
}
|
||||
if (jsonStr.endsWith('```')) {
|
||||
jsonStr = jsonStr.slice(0, -3);
|
||||
}
|
||||
jsonStr = jsonStr.trim();
|
||||
|
||||
// Find JSON object boundaries
|
||||
const startIdx = jsonStr.indexOf('{');
|
||||
const endIdx = jsonStr.lastIndexOf('}') + 1;
|
||||
|
||||
if (startIdx < 0 || endIdx <= startIdx) {
|
||||
throw new Error(`No JSON object found in response: ${fullText.substring(0, 200)}`);
|
||||
}
|
||||
|
||||
const jsonStr = fullText.substring(startIdx, endIdx);
|
||||
const parsed = JSON.parse(jsonStr);
|
||||
jsonStr = jsonStr.substring(startIdx, endIdx);
|
||||
|
||||
// Ensure numeric fields are actually numbers
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(jsonStr);
|
||||
} catch (e) {
|
||||
throw new Error(`Invalid JSON: ${jsonStr.substring(0, 200)}`);
|
||||
}
|
||||
|
||||
// Normalize response to expected format
|
||||
return {
|
||||
invoice_number: parsed.invoice_number || null,
|
||||
invoice_date: parsed.invoice_date || null,
|
||||
@@ -193,14 +223,23 @@ JSON:`;
|
||||
|
||||
/**
|
||||
* Single extraction pass: Parse with PaddleOCR-VL Full, extract with Qwen2.5 (text-only)
|
||||
* Processes ALL pages and concatenates HTML for multi-page invoice support
|
||||
*/
|
||||
async function extractOnce(images: string[], passNum: number): Promise<IInvoice> {
|
||||
// Parse document with full pipeline (PaddleOCR-VL)
|
||||
const markdown = await parseDocument(images[0]);
|
||||
console.log(` [Parse] Got ${markdown.split('\n').length} lines of Markdown`);
|
||||
// Parse ALL pages and concatenate HTML with page markers
|
||||
const htmlParts: string[] = [];
|
||||
|
||||
// Extract invoice fields from Markdown using text-only model (no images)
|
||||
return extractInvoiceFromMarkdown(markdown);
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
const pageHtml = await parseDocument(images[i]);
|
||||
// Add page marker for context
|
||||
htmlParts.push(`<!-- Page ${i + 1} -->\n${pageHtml}`);
|
||||
}
|
||||
|
||||
const fullHtml = htmlParts.join('\n\n');
|
||||
console.log(` [Parse] Got ${fullHtml.split('\n').length} lines from ${images.length} page(s)`);
|
||||
|
||||
// Extract invoice fields from HTML using text-only model (no images)
|
||||
return extractInvoiceFromHtml(fullHtml);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -438,7 +477,7 @@ tap.test('summary', async () => {
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` Invoice Extraction Summary (PaddleOCR-VL Full)`);
|
||||
console.log(`======================================================`);
|
||||
console.log(` Method: PaddleOCR-VL Full Pipeline -> Qwen2.5 (text-only)`);
|
||||
console.log(` Method: PaddleOCR-VL Full Pipeline (HTML) -> Qwen2.5 (text-only)`);
|
||||
console.log(` Passed: ${passedCount}/${totalInvoices}`);
|
||||
console.log(` Failed: ${failedCount}/${totalInvoices}`);
|
||||
console.log(` Accuracy: ${accuracy.toFixed(1)}%`);
|
||||
|
||||
352
test/test.invoices.qwen3vl.ts
Normal file
352
test/test.invoices.qwen3vl.ts
Normal file
@@ -0,0 +1,352 @@
|
||||
/**
|
||||
* Invoice extraction using Qwen3-VL-8B Vision (Direct)
|
||||
*
|
||||
* Qwen3-VL 8B is a capable vision-language model that fits in 15GB VRAM:
|
||||
* - Q4_K_M quantization (~5GB)
|
||||
* - Good balance of speed and accuracy
|
||||
*
|
||||
* Pipeline: PDF → Images → Qwen3-VL → JSON
|
||||
*/
|
||||
import { tap, expect } from '@git.zone/tstest/tapbundle';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import * as os from 'os';
|
||||
import { ensureQwen3Vl } from './helpers/docker.js';
|
||||
|
||||
const OLLAMA_URL = 'http://localhost:11434';
|
||||
const VISION_MODEL = 'qwen3-vl:8b';
|
||||
|
||||
interface IInvoice {
|
||||
invoice_number: string;
|
||||
invoice_date: string;
|
||||
vendor_name: string;
|
||||
currency: string;
|
||||
net_amount: number;
|
||||
vat_amount: number;
|
||||
total_amount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert PDF to PNG images using ImageMagick
|
||||
*/
|
||||
function convertPdfToImages(pdfPath: string): string[] {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
|
||||
const outputPattern = path.join(tempDir, 'page-%d.png');
|
||||
|
||||
try {
|
||||
// 150 DPI is sufficient for invoice extraction, reduces context size
|
||||
execSync(
|
||||
`convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
|
||||
{ stdio: 'pipe' }
|
||||
);
|
||||
|
||||
const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
|
||||
const images: string[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const imagePath = path.join(tempDir, file);
|
||||
const imageData = fs.readFileSync(imagePath);
|
||||
images.push(imageData.toString('base64'));
|
||||
}
|
||||
|
||||
return images;
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Single extraction attempt
|
||||
*/
|
||||
async function tryExtractOnce(images: string[], prompt: string): Promise<string> {
|
||||
const response = await fetch(`${OLLAMA_URL}/api/chat`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: VISION_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
images: images,
|
||||
},
|
||||
],
|
||||
stream: true,
|
||||
options: {
|
||||
num_predict: 1024,
|
||||
temperature: 0.1, // Slight randomness helps avoid stuck states
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error('No response body');
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
let fullText = '';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
const lines = chunk.split('\n').filter((l) => l.trim());
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const json = JSON.parse(line);
|
||||
if (json.message?.content) {
|
||||
fullText += json.message.content;
|
||||
}
|
||||
} catch {
|
||||
// Skip invalid JSON lines
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fullText;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract invoice data directly from images using Qwen3-VL Vision
|
||||
* Includes retry logic for empty responses
|
||||
*/
|
||||
async function extractInvoiceFromImages(images: string[]): Promise<IInvoice> {
|
||||
console.log(` [Vision] Processing ${images.length} page(s) with Qwen3-VL`);
|
||||
|
||||
// JSON schema for structured output - force the model to output valid JSON
|
||||
const invoiceSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
invoice_number: { type: 'string' },
|
||||
invoice_date: { type: 'string' },
|
||||
vendor_name: { type: 'string' },
|
||||
currency: { type: 'string' },
|
||||
net_amount: { type: 'number' },
|
||||
vat_amount: { type: 'number' },
|
||||
total_amount: { type: 'number' },
|
||||
},
|
||||
required: ['invoice_number', 'invoice_date', 'vendor_name', 'currency', 'net_amount', 'vat_amount', 'total_amount'],
|
||||
};
|
||||
|
||||
// Simple, direct prompt - don't overthink, just read the labeled fields
|
||||
const prompt = `Extract invoice data from this image. Return JSON only.
|
||||
|
||||
Find these fields:
|
||||
- invoice_number: The invoice/document number
|
||||
- invoice_date: Date in YYYY-MM-DD format
|
||||
- vendor_name: Company issuing the invoice
|
||||
- currency: EUR, USD, or GBP
|
||||
- net_amount: Amount before tax
|
||||
- vat_amount: Tax/VAT amount
|
||||
- total_amount: Final total amount
|
||||
|
||||
Return: {"invoice_number":"...", "invoice_date":"YYYY-MM-DD", "vendor_name":"...", "currency":"EUR", "net_amount":0.00, "vat_amount":0.00, "total_amount":0.00}`;
|
||||
|
||||
// Retry logic for empty responses (model sometimes returns nothing)
|
||||
const MAX_RETRIES = 3;
|
||||
let fullText = '';
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
fullText = await tryExtractOnce(images, prompt);
|
||||
|
||||
if (fullText.trim().length > 0) {
|
||||
console.log(` [Attempt ${attempt}] Got ${fullText.length} chars`);
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(` [Attempt ${attempt}] Empty response, retrying...`);
|
||||
// Small delay before retry
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
|
||||
if (fullText.trim().length === 0) {
|
||||
throw new Error(`Model returned empty response after ${MAX_RETRIES} attempts`);
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
let jsonStr = fullText.trim();
|
||||
|
||||
if (jsonStr.startsWith('```json')) jsonStr = jsonStr.slice(7);
|
||||
else if (jsonStr.startsWith('```')) jsonStr = jsonStr.slice(3);
|
||||
if (jsonStr.endsWith('```')) jsonStr = jsonStr.slice(0, -3);
|
||||
jsonStr = jsonStr.trim();
|
||||
|
||||
const startIdx = jsonStr.indexOf('{');
|
||||
const endIdx = jsonStr.lastIndexOf('}') + 1;
|
||||
|
||||
if (startIdx < 0 || endIdx <= startIdx) {
|
||||
throw new Error(`No JSON found in: ${fullText.substring(0, 500)}`);
|
||||
}
|
||||
|
||||
const extractedJson = jsonStr.substring(startIdx, endIdx);
|
||||
console.log(` [Debug] Extracted JSON: ${extractedJson.substring(0, 200)}...`);
|
||||
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(extractedJson);
|
||||
} catch (e) {
|
||||
throw new Error(`Invalid JSON: ${extractedJson.substring(0, 500)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
invoice_number: parsed.invoice_number || null,
|
||||
invoice_date: parsed.invoice_date || null,
|
||||
vendor_name: parsed.vendor_name || null,
|
||||
currency: parsed.currency || 'EUR',
|
||||
net_amount: parseFloat(parsed.net_amount) || 0,
|
||||
vat_amount: parseFloat(parsed.vat_amount) || 0,
|
||||
total_amount: parseFloat(parsed.total_amount) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize date to YYYY-MM-DD
|
||||
*/
|
||||
function normalizeDate(dateStr: string | null): string {
|
||||
if (!dateStr) return '';
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
|
||||
|
||||
const monthMap: Record<string, string> = {
|
||||
JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
|
||||
JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
|
||||
};
|
||||
|
||||
let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
|
||||
if (match) {
|
||||
return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
|
||||
if (match) {
|
||||
return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
return dateStr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare extracted vs expected
|
||||
*/
|
||||
function compareInvoice(extracted: IInvoice, expected: IInvoice): { match: boolean; errors: string[] } {
|
||||
const errors: string[] = [];
|
||||
|
||||
const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
|
||||
const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
|
||||
if (extNum !== expNum) {
|
||||
errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
|
||||
}
|
||||
|
||||
if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
|
||||
errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
|
||||
}
|
||||
|
||||
if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
|
||||
errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
|
||||
}
|
||||
|
||||
if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
|
||||
errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
|
||||
}
|
||||
|
||||
return { match: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Find test cases
|
||||
*/
|
||||
function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
|
||||
const testDir = path.join(process.cwd(), '.nogit/invoices');
|
||||
if (!fs.existsSync(testDir)) return [];
|
||||
|
||||
const files = fs.readdirSync(testDir);
|
||||
const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
|
||||
|
||||
for (const pdf of files.filter((f) => f.endsWith('.pdf'))) {
|
||||
const baseName = pdf.replace('.pdf', '');
|
||||
const jsonFile = `${baseName}.json`;
|
||||
if (files.includes(jsonFile)) {
|
||||
testCases.push({
|
||||
name: baseName,
|
||||
pdfPath: path.join(testDir, pdf),
|
||||
jsonPath: path.join(testDir, jsonFile),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return testCases.sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
// Tests
|
||||
|
||||
tap.test('setup: ensure Qwen3-VL is running', async () => {
|
||||
console.log('\n[Setup] Checking Qwen3-VL 8B (~5GB)...\n');
|
||||
const ok = await ensureQwen3Vl();
|
||||
expect(ok).toBeTrue();
|
||||
console.log('\n[Setup] Ready!\n');
|
||||
});
|
||||
|
||||
const testCases = findTestCases();
|
||||
console.log(`\nFound ${testCases.length} invoice test cases (Qwen3-VL Vision)\n`);
|
||||
|
||||
let passedCount = 0;
|
||||
let failedCount = 0;
|
||||
const times: number[] = [];
|
||||
|
||||
for (const testCase of testCases) {
|
||||
tap.test(`should extract invoice: ${testCase.name}`, async () => {
|
||||
const expected: IInvoice = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
|
||||
console.log(`\n=== ${testCase.name} ===`);
|
||||
console.log(`Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
|
||||
|
||||
const start = Date.now();
|
||||
const images = convertPdfToImages(testCase.pdfPath);
|
||||
console.log(` Pages: ${images.length}`);
|
||||
|
||||
const extracted = await extractInvoiceFromImages(images);
|
||||
console.log(` Extracted: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
|
||||
const elapsed = Date.now() - start;
|
||||
times.push(elapsed);
|
||||
|
||||
const result = compareInvoice(extracted, expected);
|
||||
|
||||
if (result.match) {
|
||||
passedCount++;
|
||||
console.log(` Result: MATCH (${(elapsed / 1000).toFixed(1)}s)`);
|
||||
} else {
|
||||
failedCount++;
|
||||
console.log(` Result: MISMATCH (${(elapsed / 1000).toFixed(1)}s)`);
|
||||
result.errors.forEach((e) => console.log(` - ${e}`));
|
||||
}
|
||||
|
||||
expect(result.match).toBeTrue();
|
||||
});
|
||||
}
|
||||
|
||||
tap.test('summary', async () => {
|
||||
const total = testCases.length;
|
||||
const accuracy = total > 0 ? (passedCount / total) * 100 : 0;
|
||||
const totalTime = times.reduce((a, b) => a + b, 0) / 1000;
|
||||
const avgTime = times.length > 0 ? totalTime / times.length : 0;
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(` Invoice Extraction Summary (Qwen3-VL Vision)`);
|
||||
console.log(`======================================================`);
|
||||
console.log(` Method: Qwen3-VL 8B (Direct Vision)`);
|
||||
console.log(` Passed: ${passedCount}/${total}`);
|
||||
console.log(` Failed: ${failedCount}/${total}`);
|
||||
console.log(` Accuracy: ${accuracy.toFixed(1)}%`);
|
||||
console.log(`------------------------------------------------------`);
|
||||
console.log(` Total time: ${totalTime.toFixed(1)}s`);
|
||||
console.log(` Avg per inv: ${avgTime.toFixed(1)}s`);
|
||||
console.log(`======================================================\n`);
|
||||
});
|
||||
|
||||
export default tap.start();
|
||||
Reference in New Issue
Block a user