4 Commits

Author SHA1 Message Date
3c5cf578a5 v1.4.0
Some checks failed
Docker (tags) / security (push) Successful in 28s
Docker (tags) / test (push) Failing after 54s
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2026-01-16 14:24:37 +00:00
82358b2d5d feat(invoices): add hybrid OCR + vision invoice/document parsing with PaddleOCR, consensus voting, and prompt/test refactors 2026-01-16 14:24:37 +00:00
acded2a165 v1.3.0
Some checks failed
Docker (tags) / security (push) Successful in 30s
Docker (tags) / test (push) Failing after 41s
Docker (tags) / release (push) Has been skipped
Docker (tags) / metadata (push) Has been skipped
2026-01-16 13:23:01 +00:00
bec379e9ca feat(paddleocr): add PaddleOCR OCR service (Docker images, server, tests, docs) and CI workflows 2026-01-16 13:23:01 +00:00
13 changed files with 1005 additions and 181 deletions

View File

@@ -0,0 +1,67 @@
name: Docker (no tags)
on:
push:
tags-ignore:
- '**'
env:
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
jobs:
security:
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
continue-on-error: true
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci command npm run build

View File

@@ -0,0 +1,101 @@
name: Docker (tags)
on:
push:
tags:
- '*'
env:
IMAGE: code.foss.global/host.today/ht-docker-node:npmci
NPMCI_COMPUTED_REPOURL: https://${{gitea.repository_owner}}:${{secrets.GITEA_TOKEN}}@gitea.lossless.digital/${{gitea.repository}}.git
NPMCI_LOGIN_DOCKER_DOCKERREGISTRY: ${{ secrets.NPMCI_LOGIN_DOCKER_DOCKERREGISTRY }}
jobs:
security:
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
continue-on-error: true
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Audit production dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --prod
continue-on-error: true
- name: Audit development dependencies
run: |
npmci command npm config set registry https://registry.npmjs.org
npmci command pnpm audit --audit-level=high --dev
continue-on-error: true
test:
needs: security
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
npmci npm prepare
- name: Test stable
run: |
npmci node install stable
npmci npm install
npmci npm test
continue-on-error: true
- name: Test build
run: |
npmci node install stable
npmci npm install
npmci command npm run build
release:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: code.foss.global/host.today/ht-docker-dbase:npmci
steps:
- uses: actions/checkout@v3
- name: Prepare
run: |
pnpm install -g pnpm
pnpm install -g @ship.zone/npmci
- name: Release
run: |
npmci docker login
npmci docker build
npmci docker push code.foss.global
metadata:
needs: test
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
container:
image: ${{ env.IMAGE }}
steps:
- uses: actions/checkout@v3
- name: Trigger
run: npmci trigger

View File

@@ -1,6 +1,6 @@
# PaddleOCR GPU Variant
# OCR processing with NVIDIA GPU support using PaddlePaddle
FROM paddlepaddle/paddle:3.0.0-gpu-cuda11.8-cudnn8.9-trt8.6
FROM paddlepaddle/paddle:2.6.2-gpu-cuda11.7-cudnn8.4-trt8.4
LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
LABEL description="PaddleOCR PP-OCRv4 - GPU optimized"
@@ -22,9 +22,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
&& rm -rf /var/lib/apt/lists/*
# Install Python dependencies
# Install Python dependencies (using stable paddleocr 2.x)
RUN pip install --no-cache-dir \
paddleocr \
paddleocr==2.8.1 \
fastapi \
uvicorn[standard] \
python-multipart \
@@ -36,10 +36,8 @@ COPY image_support_files/paddleocr_server.py /app/paddleocr_server.py
COPY image_support_files/paddleocr-entrypoint.sh /usr/local/bin/paddleocr-entrypoint.sh
RUN chmod +x /usr/local/bin/paddleocr-entrypoint.sh
# Pre-download OCR models during build (PP-OCRv4)
RUN python -c "from paddleocr import PaddleOCR; \
ocr = PaddleOCR(use_angle_cls=True, lang='en', use_gpu=False, show_log=True); \
print('English model downloaded')"
# Note: OCR models will be downloaded on first run
# This ensures compatibility across different GPU architectures
# Expose API port
EXPOSE 5000

View File

@@ -1,6 +1,6 @@
# PaddleOCR CPU Variant
# OCR processing optimized for CPU-only inference
FROM python:3.10-slim
FROM python:3.10-slim-bookworm
LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
LABEL description="PaddleOCR PP-OCRv4 - CPU optimized"
@@ -21,13 +21,14 @@ WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
libgl1-mesa-glx \
libglib2.0-0 \
libgomp1 \
curl \
&& rm -rf /var/lib/apt/lists/*
# Install Python dependencies (CPU version of PaddlePaddle)
# Install Python dependencies (CPU version of PaddlePaddle - using stable 2.x versions)
RUN pip install --no-cache-dir \
paddlepaddle \
paddleocr \
paddlepaddle==2.6.2 \
paddleocr==2.8.1 \
fastapi \
uvicorn[standard] \
python-multipart \
@@ -39,10 +40,8 @@ COPY image_support_files/paddleocr_server.py /app/paddleocr_server.py
COPY image_support_files/paddleocr-entrypoint.sh /usr/local/bin/paddleocr-entrypoint.sh
RUN chmod +x /usr/local/bin/paddleocr-entrypoint.sh
# Pre-download OCR models during build (PP-OCRv4)
RUN python -c "from paddleocr import PaddleOCR; \
ocr = PaddleOCR(use_angle_cls=True, lang='en', use_gpu=False, show_log=True); \
print('English model downloaded')"
# Note: OCR models will be downloaded on first run
# This avoids build-time segfaults with certain CPU architectures
# Expose API port
EXPOSE 5000

View File

@@ -1,5 +1,24 @@
# Changelog
## 2026-01-16 - 1.4.0 - feat(invoices)
add hybrid OCR + vision invoice/document parsing with PaddleOCR, consensus voting, and prompt/test refactors
- Add hybrid pipeline documentation and examples (PaddleOCR + MiniCPM-V) and architecture diagram in recipes/document.md
- Integrate PaddleOCR: new OCR extraction functions and OCR-only prompt flow in test/test.node.ts
- Add consensus voting and parallel-pass optimization to improve reliability (multiple passes, hashing, and majority voting)
- Refactor prompts and tests: introduce /nothink token, OCR truncation limits, separate visual and OCR-only prompts, and improved prompt building in test/test.invoices.ts
- Update image conversion defaults (200 DPI, filename change) and add TypeScript helper functions for extraction and consensus handling
## 2026-01-16 - 1.3.0 - feat(paddleocr)
add PaddleOCR OCR service (Docker images, server, tests, docs) and CI workflows
- Add GPU and CPU PaddleOCR Dockerfiles; pin paddlepaddle/paddle and paddleocr to stable 2.x and install libgomp1 for CPU builds
- Avoid pre-downloading OCR models at build-time to prevent build-time segfaults; models are downloaded on first run
- Refactor PaddleOCR FastAPI server: respect CUDA_VISIBLE_DEVICES, support per-request language, cache default language instance and create temporary instances for other languages
- Add comprehensive tests (test.paddleocr.ts) and improve invoice extraction tests (parallelize passes, JSON OCR API usage, prioritize certain test cases)
- Add Gitea CI workflows for tag and non-tag Docker runs and release pipeline (docker build/push, metadata trigger)
- Update documentation (readme.hints.md) with PaddleOCR usage and add docker registry entry to npmextra.json
## 2026-01-16 - 1.2.0 - feat(paddleocr)
add PaddleOCR support: Docker images, FastAPI server, entrypoint and tests

View File

@@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
# Environment configuration
OCR_LANGUAGE = os.environ.get('OCR_LANGUAGE', 'en')
# GPU is controlled via CUDA_VISIBLE_DEVICES environment variable
USE_GPU = os.environ.get('CUDA_VISIBLE_DEVICES', '') != '-1'
# Initialize FastAPI app
@@ -72,19 +73,29 @@ class HealthResponse(BaseModel):
gpu_enabled: bool
def get_ocr() -> PaddleOCR:
def get_ocr(lang: Optional[str] = None) -> PaddleOCR:
"""Get or initialize the OCR instance"""
global ocr_instance
if ocr_instance is None:
logger.info(f"Initializing PaddleOCR with language={OCR_LANGUAGE}, use_gpu={USE_GPU}")
ocr_instance = PaddleOCR(
use_lang = lang or OCR_LANGUAGE
# Return cached instance if same language
if ocr_instance is not None and lang is None:
return ocr_instance
logger.info(f"Initializing PaddleOCR with language={use_lang}, use_gpu={USE_GPU}")
new_ocr = PaddleOCR(
use_angle_cls=True,
lang=OCR_LANGUAGE,
lang=use_lang,
use_gpu=USE_GPU,
show_log=False
)
# Cache the default language instance
if lang is None:
ocr_instance = new_ocr
logger.info("PaddleOCR initialized successfully")
return ocr_instance
return new_ocr
def decode_base64_image(base64_string: str) -> np.ndarray:
@@ -176,19 +187,11 @@ async def ocr_base64(request: OCRRequest):
image = decode_base64_image(request.image)
# Get OCR instance (use request language if provided)
if request.language and request.language != OCR_LANGUAGE:
ocr = get_ocr(request.language)
else:
ocr = get_ocr()
# If a different language is requested, create a new instance
if request.language and request.language != OCR_LANGUAGE:
logger.info(f"Creating OCR instance for language: {request.language}")
temp_ocr = PaddleOCR(
use_angle_cls=True,
lang=request.language,
use_gpu=USE_GPU,
show_log=False
)
result = temp_ocr.ocr(image, cls=True)
else:
result = ocr.ocr(image, cls=True)
# Process results
@@ -228,19 +231,11 @@ async def ocr_upload(
image_array = np.array(image)
# Get OCR instance
if language and language != OCR_LANGUAGE:
ocr = get_ocr(language)
else:
ocr = get_ocr()
# If a different language is requested, create a new instance
if language and language != OCR_LANGUAGE:
logger.info(f"Creating OCR instance for language: {language}")
temp_ocr = PaddleOCR(
use_angle_cls=True,
lang=language,
use_gpu=USE_GPU,
show_log=False
)
result = temp_ocr.ocr(image_array, cls=True)
else:
result = ocr.ocr(image_array, cls=True)
# Process results

View File

@@ -1,7 +1,10 @@
{
"npmci": {
"npmGlobalTools": [],
"npmAccessLevel": "public"
"npmAccessLevel": "public",
"dockerRegistries": [
"code.foss.global"
]
},
"gitzone": {
"projectType": "docker",

View File

@@ -1,6 +1,6 @@
{
"name": "@host.today/ht-docker-ai",
"version": "1.2.0",
"version": "1.4.0",
"type": "module",
"private": false,
"description": "Docker images for AI vision-language models including MiniCPM-V 4.5",

View File

@@ -77,6 +77,81 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
CPU variant has longer `start-period` (120s) due to slower startup.
## PaddleOCR
### Overview
PaddleOCR is a standalone OCR service using PaddlePaddle's PP-OCRv4 model. It provides:
- Text detection and recognition
- Multi-language support
- FastAPI REST API
- GPU and CPU variants
### Docker Images
| Tag | Description |
|-----|-------------|
| `paddleocr` | GPU variant (default) |
| `paddleocr-gpu` | GPU variant (alias) |
| `paddleocr-cpu` | CPU-only variant |
### API Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/health` | GET | Health check with model info |
| `/ocr` | POST | OCR with base64 image (JSON body) |
| `/ocr/upload` | POST | OCR with file upload (multipart form) |
### Request/Response Format
**POST /ocr (JSON)**
```json
{
"image": "<base64-encoded-image>",
"language": "en" // optional
}
```
**POST /ocr/upload (multipart)**
- `img`: image file
- `language`: optional language code
**Response**
```json
{
"success": true,
"results": [
{
"text": "Invoice #12345",
"confidence": 0.98,
"box": [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
}
]
}
```
### Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `OCR_LANGUAGE` | `en` | Default language for OCR |
| `SERVER_PORT` | `5000` | Server port |
| `SERVER_HOST` | `0.0.0.0` | Server host |
| `CUDA_VISIBLE_DEVICES` | (auto) | Set to `-1` for CPU-only |
### Performance
- **GPU**: ~1-3 seconds per page
- **CPU**: ~10-30 seconds per page
### Supported Languages
Common language codes: `en` (English), `ch` (Chinese), `de` (German), `fr` (French), `es` (Spanish), `ja` (Japanese), `ko` (Korean)
---
## Adding New Models
To add a new model variant:

View File

@@ -1,129 +1,250 @@
# Bank Statement Parsing with MiniCPM-V 4.5
# Document Recognition with Hybrid OCR + Vision AI
Recipe for extracting transactions from bank statement PDFs using vision-language AI.
Recipe for extracting structured data from invoices and documents using a hybrid approach:
PaddleOCR for text extraction + MiniCPM-V 4.5 for intelligent parsing.
## Model
## Architecture
- **Model**: MiniCPM-V 4.5 (8B parameters)
- **Ollama Name**: `openbmb/minicpm-v4.5:q8_0`
- **Quantization**: Q8_0 (9.8GB VRAM)
- **Runtime**: Ollama on GPU
```
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
│ PDF/Image │ ───> │ PaddleOCR │ ───> │ Raw Text │
└──────────────┘ └──────────────┘ └──────┬───────┘
┌──────────────┐ │
│ MiniCPM-V │ <───────────┘
│ 4.5 VLM │ <─── Image
└──────┬───────┘
┌──────▼───────┐
│ Structured │
│ JSON │
└──────────────┘
```
## Why Hybrid?
| Approach | Accuracy | Speed | Best For |
|----------|----------|-------|----------|
| VLM Only | 85-90% | Fast | Simple layouts |
| OCR Only | N/A | Fast | Just text extraction |
| **Hybrid** | **91%+** | Medium | Complex invoices |
The hybrid approach provides OCR text as context to the VLM, improving accuracy on:
- Small text and numbers
- Low contrast documents
- Dense tables
## Services
| Service | Port | Purpose |
|---------|------|---------|
| PaddleOCR | 5000 | Text extraction |
| Ollama (MiniCPM-V) | 11434 | Intelligent parsing |
## Running the Containers
**Start both services:**
```bash
# PaddleOCR (CPU is sufficient for OCR)
docker run -d --name paddleocr -p 5000:5000 \
code.foss.global/host.today/ht-docker-ai:paddleocr-cpu
# MiniCPM-V 4.5 (GPU recommended)
docker run -d --name minicpm --gpus all -p 11434:11434 \
-v ollama-data:/root/.ollama \
code.foss.global/host.today/ht-docker-ai:minicpm45v
```
## Image Conversion
Convert PDF to PNG at 300 DPI for optimal OCR accuracy.
Convert PDF to PNG at 200 DPI:
```bash
convert -density 300 -quality 100 input.pdf \
convert -density 200 -quality 90 input.pdf \
-background white -alpha remove \
output-%d.png
page-%d.png
```
**Parameters:**
- `-density 300`: 300 DPI resolution (critical for accuracy)
- `-quality 100`: Maximum quality
- `-background white -alpha remove`: Remove transparency
- `output-%d.png`: Outputs page-0.png, page-1.png, etc.
## Step 1: Extract OCR Text
**Dependencies:**
```bash
apt-get install imagemagick
```typescript
async function extractOcrText(imageBase64: string): Promise<string> {
const response = await fetch('http://localhost:5000/ocr', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
const data = await response.json();
if (data.success && data.results) {
return data.results.map((r: { text: string }) => r.text).join('\n');
}
return '';
}
```
## Prompt
## Step 2: Build Enhanced Prompt
```
You are a bank statement parser. Extract EVERY transaction from the table.
```typescript
function buildPrompt(ocrText: string): string {
const base = `You are an invoice parser. Extract the following fields:
Read the Amount column carefully:
- "- 21,47 €" means DEBIT, output as: -21.47
- "+ 1.000,00 €" means CREDIT, output as: 1000.00
- European format: comma = decimal point
1. invoice_number: The invoice/receipt number
2. invoice_date: Date in YYYY-MM-DD format
3. vendor_name: Company that issued the invoice
4. currency: EUR, USD, etc.
5. net_amount: Amount before tax (if shown)
6. vat_amount: Tax/VAT amount (0 if reverse charge)
7. total_amount: Final amount due
For each row output: {"date":"YYYY-MM-DD","counterparty":"NAME","amount":-21.47}
Return ONLY valid JSON:
{"invoice_number":"XXX","invoice_date":"YYYY-MM-DD","vendor_name":"Company","currency":"EUR","net_amount":100.00,"vat_amount":19.00,"total_amount":119.00}`;
Do not skip any rows. Return complete JSON array:
if (ocrText) {
return `${base}
OCR text extracted from the invoice:
---
${ocrText}
---
Cross-reference the image with the OCR text above for accuracy.`;
}
return base;
}
```
## API Call
## Step 3: Call Vision-Language Model
```python
import base64
import requests
```typescript
async function extractInvoice(images: string[], ocrText: string): Promise<Invoice> {
const payload = {
model: 'openbmb/minicpm-v4.5:q8_0',
prompt: buildPrompt(ocrText),
images, // Base64 encoded
stream: false,
options: {
num_predict: 2048,
temperature: 0.1,
},
};
# Load images
with open('page-0.png', 'rb') as f:
page0 = base64.b64encode(f.read()).decode('utf-8')
with open('page-1.png', 'rb') as f:
page1 = base64.b64encode(f.read()).decode('utf-8')
const response = await fetch('http://localhost:11434/api/generate', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload),
});
payload = {
"model": "openbmb/minicpm-v4.5:q8_0",
"prompt": prompt,
"images": [page0, page1], # Multiple pages supported
"stream": False,
"options": {
"num_predict": 16384,
"temperature": 0.1
const result = await response.json();
return JSON.parse(result.response);
}
```
## Consensus Voting
For production reliability, run multiple extraction passes and require consensus:
```typescript
async function extractWithConsensus(images: string[], maxPasses: number = 5): Promise<Invoice> {
const results: Map<string, { invoice: Invoice; count: number }> = new Map();
// Optimization: Run Pass 1 (no OCR) parallel with OCR + Pass 2
const [pass1Result, ocrText] = await Promise.all([
extractInvoice(images, ''),
extractOcrText(images[0]),
]);
// Add Pass 1 result
addResult(results, pass1Result);
// Pass 2 with OCR context
const pass2Result = await extractInvoice(images, ocrText);
addResult(results, pass2Result);
// Check for consensus (2 matching results)
for (const [hash, data] of results) {
if (data.count >= 2) {
return data.invoice; // Consensus reached!
}
}
response = requests.post(
'http://localhost:11434/api/generate',
json=payload,
timeout=600
)
// Continue until consensus or max passes
for (let pass = 3; pass <= maxPasses; pass++) {
const result = await extractInvoice(images, ocrText);
addResult(results, result);
// Check consensus...
}
result = response.json()['response']
// Return most common result
return getMostCommon(results);
}
function hashInvoice(inv: Invoice): string {
return `${inv.invoice_number}|${inv.invoice_date}|${inv.total_amount.toFixed(2)}`;
}
```
## Output Format
```json
[
{"date":"2022-04-01","counterparty":"DIGITALOCEAN.COM","amount":-21.47},
{"date":"2022-04-01","counterparty":"DIGITALOCEAN.COM","amount":-58.06},
{"date":"2022-04-12","counterparty":"LOSSLESS GMBH","amount":1000.00}
]
{
"invoice_number": "INV-2024-001234",
"invoice_date": "2024-08-15",
"vendor_name": "Hetzner Online GmbH",
"currency": "EUR",
"net_amount": 167.52,
"vat_amount": 31.83,
"total_amount": 199.35
}
```
## Running the Container
**GPU (recommended):**
```bash
docker run -d --gpus all -p 11434:11434 \
-v ollama-data:/root/.ollama \
-e MODEL_NAME="openbmb/minicpm-v4.5:q8_0" \
ht-docker-ai:minicpm45v
```
**CPU (slower):**
```bash
docker run -d -p 11434:11434 \
-v ollama-data:/root/.ollama \
-e MODEL_NAME="openbmb/minicpm-v4.5:q4_0" \
ht-docker-ai:minicpm45v-cpu
```
## Hardware Requirements
| Quantization | VRAM/RAM | Speed |
|--------------|----------|-------|
| Q8_0 (GPU) | 10GB | Fast |
| Q4_0 (CPU) | 8GB | Slow |
## Test Results
| Statement | Pages | Transactions | Accuracy |
|-----------|-------|--------------|----------|
| bunq-2022-04 | 2 | 26 | 100% |
| bunq-2021-06 | 3 | 28 | 100% |
Tested on 46 real invoices from various vendors:
| Metric | Value |
|--------|-------|
| **Accuracy** | 91.3% (42/46) |
| **Avg Time** | 42.7s per invoice |
| **Consensus Rate** | 85% in 2 passes |
### Per-Vendor Results
| Vendor | Invoices | Accuracy |
|--------|----------|----------|
| Hetzner | 3 | 100% |
| DigitalOcean | 4 | 100% |
| Adobe | 3 | 100% |
| Cloudflare | 1 | 100% |
| Wasabi | 4 | 100% |
| Figma | 3 | 100% |
| Google Cloud | 1 | 100% |
| MongoDB | 3 | 0% (date parsing) |
## Hardware Requirements
| Component | Minimum | Recommended |
|-----------|---------|-------------|
| PaddleOCR (CPU) | 4GB RAM | 8GB RAM |
| MiniCPM-V (GPU) | 10GB VRAM | 12GB VRAM |
| MiniCPM-V (CPU) | 16GB RAM | 32GB RAM |
## Tips
1. **DPI matters**: 150 DPI causes missed rows; 300 DPI is optimal
2. **PNG over JPEG**: PNG preserves text clarity better
3. **Remove alpha**: Some models struggle with transparency
4. **Multi-page**: Pass all pages in single request for context
1. **Use hybrid approach**: OCR text dramatically improves number/date accuracy
2. **Consensus voting**: Run 2-5 passes to catch hallucinations
3. **200 DPI is optimal**: Higher doesn't help, lower loses detail
4. **PNG over JPEG**: Preserves text clarity
5. **Temperature 0.1**: Low temperature for consistent output
6. **European format**: Explicitly explain comma=decimal in prompt
6. **Multi-page support**: Pass all pages in single request for context
7. **Normalize for comparison**: Ignore case/whitespace when comparing invoice numbers
## Common Issues
| Issue | Cause | Solution |
|-------|-------|----------|
| Wrong date | Multiple dates on invoice | Be specific in prompt about which date |
| Wrong currency | Symbol vs code mismatch | OCR helps disambiguate |
| Missing digits | Low resolution | Increase density to 300 DPI |
| Hallucinated data | VLM uncertainty | Use consensus voting |

View File

@@ -22,16 +22,11 @@ interface IInvoice {
* Extract OCR text from an image using PaddleOCR
*/
async function extractOcrText(imageBase64: string): Promise<string> {
const formData = new FormData();
const imageBuffer = Buffer.from(imageBase64, 'base64');
const blob = new Blob([imageBuffer], { type: 'image/png' });
formData.append('img', blob, 'image.png');
formData.append('outtype', 'json');
try {
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
body: formData,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
if (!response.ok) return '';
@@ -50,7 +45,8 @@ async function extractOcrText(imageBase64: string): Promise<string> {
* Build prompt with optional OCR text
*/
function buildPrompt(ocrText: string): string {
const base = `You are an invoice parser. Extract the following fields from this invoice:
const base = `/nothink
You are an invoice parser. Extract the following fields from this invoice:
1. invoice_number: The invoice/receipt number
2. invoice_date: Date in YYYY-MM-DD format
@@ -67,11 +63,17 @@ If a field is not visible, use null for strings or 0 for numbers.
No explanation, just the JSON object.`;
if (ocrText) {
// Limit OCR text to prevent context overflow
const maxOcrLength = 4000;
const truncatedOcr = ocrText.length > maxOcrLength
? ocrText.substring(0, maxOcrLength) + '\n... (truncated)'
: ocrText;
return `${base}
OCR text extracted from the invoice:
OCR text extracted from the invoice (use for reference):
---
${ocrText}
${truncatedOcr}
---
Cross-reference the image with the OCR text above for accuracy.`;
@@ -180,29 +182,64 @@ function hashInvoice(invoice: IInvoice): string {
/**
* Extract with majority voting - run until 2 passes match
* Optimization: Run Pass 1, OCR, and Pass 2 (after OCR) in parallel
*/
async function extractWithConsensus(images: string[], invoiceName: string, maxPasses: number = 5): Promise<IInvoice> {
const results: Array<{ invoice: IInvoice; hash: string }> = [];
const hashCounts: Map<string, number> = new Map();
// Extract OCR text from first page
const ocrText = await extractOcrText(images[0]);
const addResult = (invoice: IInvoice, passLabel: string): number => {
const hash = hashInvoice(invoice);
results.push({ invoice, hash });
hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);
console.log(` [${passLabel}] ${invoice.invoice_number} | ${invoice.invoice_date} | ${invoice.total_amount} ${invoice.currency}`);
return hashCounts.get(hash)!;
};
// OPTIMIZATION: Run Pass 1 (no OCR) in parallel with OCR -> Pass 2 (with OCR)
let ocrText = '';
const pass1Promise = extractOnce(images, 1, '').catch((err) => ({ error: err }));
// OCR then immediately Pass 2
const ocrThenPass2Promise = (async () => {
ocrText = await extractOcrText(images[0]);
if (ocrText) {
console.log(` [OCR] Extracted ${ocrText.split('\n').length} text lines`);
}
return extractOnce(images, 2, ocrText).catch((err) => ({ error: err }));
})();
for (let pass = 1; pass <= maxPasses; pass++) {
// Wait for both to complete
const [pass1Result, pass2Result] = await Promise.all([pass1Promise, ocrThenPass2Promise]);
// Process Pass 1 result
if ('error' in pass1Result) {
console.log(` [Pass 1] Error: ${(pass1Result as {error: unknown}).error}`);
} else {
const count = addResult(pass1Result as IInvoice, 'Pass 1');
if (count >= 2) {
console.log(` [Consensus] Reached after parallel passes`);
return pass1Result as IInvoice;
}
}
// Process Pass 2 result
if ('error' in pass2Result) {
console.log(` [Pass 2+OCR] Error: ${(pass2Result as {error: unknown}).error}`);
} else {
const count = addResult(pass2Result as IInvoice, 'Pass 2+OCR');
if (count >= 2) {
console.log(` [Consensus] Reached after parallel passes`);
return pass2Result as IInvoice;
}
}
// Continue with passes 3+ using OCR text if no consensus yet
for (let pass = 3; pass <= maxPasses; pass++) {
try {
const invoice = await extractOnce(images, pass, ocrText);
const hash = hashInvoice(invoice);
const count = addResult(invoice, `Pass ${pass}+OCR`);
results.push({ invoice, hash });
hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);
console.log(` [Pass ${pass}] ${invoice.invoice_number} | ${invoice.invoice_date} | ${invoice.total_amount} ${invoice.currency}`);
// Check if we have consensus (2+ matching)
const count = hashCounts.get(hash)!;
if (count >= 2) {
console.log(` [Consensus] Reached after ${pass} passes`);
return invoice;
@@ -267,6 +304,7 @@ function compareInvoice(
/**
* Find all test cases (PDF + JSON pairs) in .nogit/invoices/
* Priority invoices (like vodafone) run first for quick feedback
*/
function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
const testDir = path.join(process.cwd(), '.nogit/invoices');
@@ -290,6 +328,22 @@ function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: strin
}
}
// Sort with priority invoices first, then alphabetically
const priorityPrefixes = ['vodafone'];
testCases.sort((a, b) => {
const aPriority = priorityPrefixes.findIndex((p) => a.name.startsWith(p));
const bPriority = priorityPrefixes.findIndex((p) => b.name.startsWith(p));
// Both have priority - sort by priority order
if (aPriority >= 0 && bPriority >= 0) return aPriority - bPriority;
// Only a has priority - a comes first
if (aPriority >= 0) return -1;
// Only b has priority - b comes first
if (bPriority >= 0) return 1;
// Neither has priority - alphabetical
return a.name.localeCompare(b.name);
});
return testCases;
}

View File

@@ -6,8 +6,11 @@ import * as os from 'os';
const OLLAMA_URL = 'http://localhost:11434';
const MODEL = 'openbmb/minicpm-v4.5:q8_0';
const PADDLEOCR_URL = 'http://localhost:5000';
const EXTRACT_PROMPT = `You are a bank statement parser. Extract EVERY transaction from the table.
// Prompt for visual extraction (with images)
const VISUAL_EXTRACT_PROMPT = `/nothink
You are a bank statement parser. Extract EVERY transaction from the table.
Read the Amount column carefully:
- "- 21,47 €" means DEBIT, output as: -21.47
@@ -18,6 +21,60 @@ For each row output: {"date":"YYYY-MM-DD","counterparty":"NAME","amount":-21.47}
Do not skip any rows. Return ONLY the JSON array, no explanation.`;
// Prompt for OCR-only extraction (no images)
const OCR_EXTRACT_PROMPT = `/nothink
You are a bank statement parser. Extract EVERY transaction from the OCR text below.
Read the Amount values carefully:
- "- 21,47 €" means DEBIT, output as: -21.47
- "+ 1.000,00 €" means CREDIT, output as: 1000.00
- European format: comma = decimal point
For each transaction output: {"date":"YYYY-MM-DD","counterparty":"NAME","amount":-21.47}
Do not skip any transactions. Return ONLY the JSON array, no explanation.`;
/**
* Build prompt for OCR-only extraction (no images)
*/
function buildOcrOnlyPrompt(ocrText: string): string {
// Limit OCR text to prevent context overflow
const maxOcrLength = 12000;
const truncatedOcr = ocrText.length > maxOcrLength
? ocrText.substring(0, maxOcrLength) + '\n... (truncated)'
: ocrText;
return `${OCR_EXTRACT_PROMPT}
OCR text from bank statement:
---
${truncatedOcr}
---`;
}
/**
* Extract OCR text from an image using PaddleOCR
*/
async function extractOcrText(imageBase64: string): Promise<string> {
try {
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
if (!response.ok) return '';
const data = await response.json();
if (data.success && data.results) {
return data.results.map((r: { text: string }) => r.text).join('\n');
}
} catch {
// PaddleOCR unavailable
}
return '';
}
interface ITransaction {
date: string;
counterparty: string;
@@ -53,12 +110,12 @@ function convertPdfToImages(pdfPath: string): string[] {
}
/**
* Single extraction pass
* Visual extraction pass (with images)
*/
async function extractOnce(images: string[], passNum: number): Promise<ITransaction[]> {
async function extractVisual(images: string[], passLabel: string): Promise<ITransaction[]> {
const payload = {
model: MODEL,
prompt: EXTRACT_PROMPT,
prompt: VISUAL_EXTRACT_PROMPT,
images,
stream: true,
options: {
@@ -67,6 +124,31 @@ async function extractOnce(images: string[], passNum: number): Promise<ITransact
},
};
return doExtraction(payload, passLabel);
}
/**
* OCR-only extraction pass (no images, just text)
*/
async function extractFromOcr(ocrText: string, passLabel: string): Promise<ITransaction[]> {
const payload = {
model: MODEL,
prompt: buildOcrOnlyPrompt(ocrText),
stream: true,
options: {
num_predict: 16384,
temperature: 0.1,
},
};
return doExtraction(payload, passLabel);
}
/**
* Common extraction logic
*/
async function doExtraction(payload: object, passLabel: string): Promise<ITransaction[]> {
const response = await fetch(`${OLLAMA_URL}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -86,7 +168,7 @@ async function extractOnce(images: string[], passNum: number): Promise<ITransact
let fullText = '';
let lineBuffer = '';
console.log(`[Pass ${passNum}] Extracting...`);
console.log(`[${passLabel}] Extracting...`);
while (true) {
const { done, value } = await reader.read();
@@ -144,30 +226,78 @@ function hashTransactions(transactions: ITransaction[]): string {
/**
* Extract with majority voting - run until 2 passes match
* Strategy: Pass 1 = Visual (images), Pass 2 = OCR-only (text), Pass 3+ = Visual
*/
async function extractWithConsensus(images: string[], maxPasses: number = 5): Promise<ITransaction[]> {
const results: Array<{ transactions: ITransaction[]; hash: string }> = [];
const hashCounts: Map<string, number> = new Map();
for (let pass = 1; pass <= maxPasses; pass++) {
const transactions = await extractOnce(images, pass);
const addResult = (transactions: ITransaction[], passLabel: string): number => {
const hash = hashTransactions(transactions);
results.push({ transactions, hash });
hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);
console.log(`[${passLabel}] Got ${transactions.length} transactions (hash: ${hash.substring(0, 20)}...)`);
return hashCounts.get(hash)!;
};
console.log(`[Pass ${pass}] Got ${transactions.length} transactions (hash: ${hash.substring(0, 20)}...)`);
// Run Pass 1 (Visual) in parallel with OCR extraction
let ocrText = '';
const pass1Promise = extractVisual(images, 'Pass 1 Visual').catch((err) => ({ error: err }));
// Check if we have consensus (2+ matching)
const count = hashCounts.get(hash)!;
// Extract OCR from all pages
const ocrPromise = (async () => {
const ocrTexts: string[] = [];
for (let i = 0; i < images.length; i++) {
const pageOcr = await extractOcrText(images[i]);
if (pageOcr) {
ocrTexts.push(`--- Page ${i + 1} ---\n${pageOcr}`);
}
}
ocrText = ocrTexts.join('\n\n');
if (ocrText) {
console.log(`[OCR] Extracted text from ${ocrTexts.length} page(s)`);
}
return ocrText;
})();
// Wait for Pass 1 and OCR to complete
const [pass1Result] = await Promise.all([pass1Promise, ocrPromise]);
// Process Pass 1 result
if ('error' in pass1Result) {
console.log(`[Pass 1] Error: ${(pass1Result as { error: unknown }).error}`);
} else {
addResult(pass1Result as ITransaction[], 'Pass 1 Visual');
}
// Pass 2: OCR-only (no images) - faster, different approach
if (ocrText) {
try {
const pass2Result = await extractFromOcr(ocrText, 'Pass 2 OCR-only');
const count = addResult(pass2Result, 'Pass 2 OCR-only');
if (count >= 2) {
console.log(`[Consensus] Reached after ${pass} passes (${count} matching results)`);
console.log(`[Consensus] Visual and OCR extractions match!`);
return pass2Result;
}
} catch (err) {
console.log(`[Pass 2 OCR-only] Error: ${err}`);
}
}
// Continue with visual passes 3+ if no consensus yet
for (let pass = 3; pass <= maxPasses; pass++) {
try {
const transactions = await extractVisual(images, `Pass ${pass} Visual`);
const count = addResult(transactions, `Pass ${pass} Visual`);
if (count >= 2) {
console.log(`[Consensus] Reached after ${pass} passes`);
return transactions;
}
// After 2 passes, if no match yet, continue
if (pass >= 2) {
console.log(`[Pass ${pass}] No consensus yet, trying again...`);
} catch (err) {
console.log(`[Pass ${pass}] Error: ${err}`);
}
}
@@ -181,6 +311,10 @@ async function extractWithConsensus(images: string[], maxPasses: number = 5): Pr
}
}
if (!bestHash) {
throw new Error('No valid results obtained');
}
const best = results.find((r) => r.hash === bestHash)!;
console.log(`[No consensus] Using most common result (${bestCount}/${maxPasses} passes)`);
return best.transactions;

258
test/test.paddleocr.ts Normal file
View File

@@ -0,0 +1,258 @@
import { tap, expect } from '@git.zone/tstest/tapbundle';
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import * as os from 'os';
const PADDLEOCR_URL = 'http://localhost:5000';
interface IOCRResult {
text: string;
confidence: number;
box: number[][];
}
interface IOCRResponse {
success: boolean;
results: IOCRResult[];
error?: string;
}
interface IHealthResponse {
status: string;
model: string;
language: string;
gpu_enabled: boolean;
}
/**
* Convert PDF first page to PNG using ImageMagick
*/
function convertPdfToImage(pdfPath: string): string {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
const outputPath = path.join(tempDir, 'page.png');
try {
execSync(
`convert -density 200 -quality 90 "${pdfPath}[0]" -background white -alpha remove "${outputPath}"`,
{ stdio: 'pipe' }
);
const imageData = fs.readFileSync(outputPath);
return imageData.toString('base64');
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
/**
* Create a simple test image with text using ImageMagick
*/
function createTestImage(text: string): string {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-image-'));
const outputPath = path.join(tempDir, 'test.png');
try {
execSync(
`convert -size 400x100 xc:white -font DejaVu-Sans -pointsize 24 -fill black -gravity center -annotate 0 "${text}" "${outputPath}"`,
{ stdio: 'pipe' }
);
const imageData = fs.readFileSync(outputPath);
return imageData.toString('base64');
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
}
// Health check test
tap.test('should respond to health check', async () => {
const response = await fetch(`${PADDLEOCR_URL}/health`);
expect(response.ok).toBeTrue();
const data: IHealthResponse = await response.json();
expect(data.status).toEqual('healthy');
expect(data.model).toEqual('PP-OCRv4');
expect(data.language).toBeTypeofString();
expect(data.gpu_enabled).toBeTypeofBoolean();
console.log(`PaddleOCR Status: ${data.status}`);
console.log(` Model: ${data.model}`);
console.log(` Language: ${data.language}`);
console.log(` GPU Enabled: ${data.gpu_enabled}`);
});
// Base64 OCR test
tap.test('should perform OCR on base64 image', async () => {
// Create a test image with known text
const testText = 'Hello World 12345';
console.log(`Creating test image with text: "${testText}"`);
const imageBase64 = createTestImage(testText);
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
expect(response.ok).toBeTrue();
const data: IOCRResponse = await response.json();
expect(data.success).toBeTrue();
expect(data.results).toBeArray();
const extractedText = data.results.map((r) => r.text).join(' ');
console.log(`Extracted text: "${extractedText}"`);
// Check that we got some text back
expect(data.results.length).toBeGreaterThan(0);
// Check that at least some of the expected text was found
const normalizedExtracted = extractedText.toLowerCase().replace(/\s+/g, '');
const normalizedExpected = testText.toLowerCase().replace(/\s+/g, '');
const hasPartialMatch =
normalizedExtracted.includes('hello') ||
normalizedExtracted.includes('world') ||
normalizedExtracted.includes('12345');
expect(hasPartialMatch).toBeTrue();
});
// File upload OCR test
tap.test('should perform OCR via file upload', async () => {
const testText = 'Invoice Number 98765';
console.log(`Creating test image with text: "${testText}"`);
const imageBase64 = createTestImage(testText);
const imageBuffer = Buffer.from(imageBase64, 'base64');
const formData = new FormData();
const blob = new Blob([imageBuffer], { type: 'image/png' });
formData.append('img', blob, 'test.png');
const response = await fetch(`${PADDLEOCR_URL}/ocr/upload`, {
method: 'POST',
body: formData,
});
expect(response.ok).toBeTrue();
const data: IOCRResponse = await response.json();
expect(data.success).toBeTrue();
expect(data.results).toBeArray();
const extractedText = data.results.map((r) => r.text).join(' ');
console.log(`Extracted text: "${extractedText}"`);
// Check that we got some text back
expect(data.results.length).toBeGreaterThan(0);
});
// OCR result structure test
tap.test('should return proper OCR result structure', async () => {
const testText = 'Test 123';
const imageBase64 = createTestImage(testText);
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
const data: IOCRResponse = await response.json();
if (data.results.length > 0) {
const result = data.results[0];
// Check result has required fields
expect(result.text).toBeTypeofString();
expect(result.confidence).toBeTypeofNumber();
expect(result.box).toBeArray();
// Check bounding box structure (4 points, each with x,y)
expect(result.box.length).toEqual(4);
for (const point of result.box) {
expect(point.length).toEqual(2);
expect(point[0]).toBeTypeofNumber();
expect(point[1]).toBeTypeofNumber();
}
// Confidence should be between 0 and 1
expect(result.confidence).toBeGreaterThan(0);
expect(result.confidence).toBeLessThanOrEqual(1);
console.log(`Result structure valid:`);
console.log(` Text: "${result.text}"`);
console.log(` Confidence: ${(result.confidence * 100).toFixed(1)}%`);
console.log(` Box: ${JSON.stringify(result.box)}`);
}
});
// Test with actual invoice if available
const invoiceDir = path.join(process.cwd(), '.nogit/invoices');
if (fs.existsSync(invoiceDir)) {
const pdfFiles = fs.readdirSync(invoiceDir).filter((f) => f.endsWith('.pdf'));
if (pdfFiles.length > 0) {
const testPdf = pdfFiles[0];
tap.test(`should extract text from invoice: ${testPdf}`, async () => {
const pdfPath = path.join(invoiceDir, testPdf);
console.log(`Converting ${testPdf} to image...`);
const imageBase64 = convertPdfToImage(pdfPath);
console.log(`Image size: ${(imageBase64.length / 1024).toFixed(1)} KB`);
const startTime = Date.now();
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: imageBase64 }),
});
const endTime = Date.now();
const elapsedMs = endTime - startTime;
expect(response.ok).toBeTrue();
const data: IOCRResponse = await response.json();
expect(data.success).toBeTrue();
console.log(`OCR completed in ${(elapsedMs / 1000).toFixed(2)}s`);
console.log(`Found ${data.results.length} text regions`);
// Print first 10 results
const preview = data.results.slice(0, 10);
console.log(`\nFirst ${preview.length} results:`);
for (const result of preview) {
console.log(` [${(result.confidence * 100).toFixed(0)}%] ${result.text}`);
}
if (data.results.length > 10) {
console.log(` ... and ${data.results.length - 10} more`);
}
// Should find text in an invoice
expect(data.results.length).toBeGreaterThan(5);
});
}
}
// Error handling test
tap.test('should handle invalid base64 gracefully', async () => {
const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ image: 'not-valid-base64!!!' }),
});
const data: IOCRResponse = await response.json();
// Should return success: false with error message
expect(data.success).toBeFalse();
expect(data.error).toBeTypeofString();
console.log(`Error handling works: ${data.error}`);
});
export default tap.start();