v1.15.1

fix(tests): enable progress events in invoice tests and bump @push.rocks/smartagent devDependency to ^1.5.4
chore: update smartagent to v1.5.2 (streaming support for native tool calling)
2026-01-20 03:19:58 +00:00 · 2026-01-20 03:19:58 +00:00 · 2026-01-20 02:55:28 +00:00 · 2026-01-20 02:51:52 +00:00 · 2026-01-20 01:17:41 +00:00 · 2026-01-20 01:17:41 +00:00
27 changed files with 6605 additions and 2394 deletions
--- a/27
+++ b/27
@@ -1,27 +0,0 @@
 # MiniCPM-V 4.5 CPU Variant
 # Vision-Language Model optimized for CPU-only inference
 FROM ollama/ollama:latest
 LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
 LABEL description="MiniCPM-V 4.5 Vision-Language Model - CPU optimized (GGUF)"
 LABEL org.opencontainers.image.source="https://code.foss.global/host.today/ht-docker-ai"
 # Environment configuration for CPU-only mode
 ENV MODEL_NAME="minicpm-v"
 ENV OLLAMA_HOST="0.0.0.0"
 ENV OLLAMA_ORIGINS="*"
 # Disable GPU usage for CPU-only variant
 ENV CUDA_VISIBLE_DEVICES=""
 # Copy and setup entrypoint
 COPY image_support_files/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 # Expose Ollama API port
 EXPOSE 11434
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
    CMD curl -f http://localhost:11434/api/tags || exit 1
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
--- a/2
+++ b/2
@@ -12,7 +12,7 @@ ENV OLLAMA_HOST="0.0.0.0"
 ENV OLLAMA_ORIGINS="*"
 # Copy and setup entrypoint
-COPY image_support_files/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY image_support_files/minicpm45v_entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 # Expose Ollama API port
--- a/34
+++ b/34
@@ -0,0 +1,34 @@
 # Nanonets-OCR2-3B Vision Language Model
 # Based on Qwen2.5-VL-3B, fine-tuned for document OCR (Oct 2025 release)
 # Improvements over OCR-s: better semantic tagging, LaTeX equations, flowcharts
 # ~12-16GB VRAM with 30K context, outputs structured markdown with semantic tags
 #
 # Build: docker build -f Dockerfile_nanonets_vllm_gpu_VRAM10GB -t nanonets-ocr .
 # Run:   docker run --gpus all -p 8000:8000 -v ht-huggingface-cache:/root/.cache/huggingface nanonets-ocr
 FROM vllm/vllm-openai:latest
 LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
 LABEL description="Nanonets-OCR2-3B - Document OCR optimized Vision Language Model"
 LABEL org.opencontainers.image.source="https://code.foss.global/host.today/ht-docker-ai"
 # Environment configuration
 ENV MODEL_NAME="nanonets/Nanonets-OCR2-3B"
 ENV HOST="0.0.0.0"
 ENV PORT="8000"
 ENV MAX_MODEL_LEN="30000"
 ENV GPU_MEMORY_UTILIZATION="0.9"
 # Expose OpenAI-compatible API port
 EXPOSE 8000
 # Health check - vLLM exposes /health endpoint
 HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=5 \
    CMD curl -f http://localhost:8000/health || exit 1
 # Start vLLM server with Nanonets-OCR2-3B model
 CMD ["--model", "nanonets/Nanonets-OCR2-3B", \
     "--trust-remote-code", \
     "--max-model-len", "30000", \
     "--host", "0.0.0.0", \
     "--port", "8000"]
--- a/49
+++ b/49
@@ -1,49 +0,0 @@
 # PaddleOCR GPU Variant
 # OCR processing with NVIDIA GPU support using PaddlePaddle
 FROM paddlepaddle/paddle:2.6.2-gpu-cuda11.7-cudnn8.4-trt8.4
 LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
 LABEL description="PaddleOCR PP-OCRv4 - GPU optimized"
 LABEL org.opencontainers.image.source="https://code.foss.global/host.today/ht-docker-ai"
 # Environment configuration
 ENV OCR_LANGUAGE="en"
 ENV SERVER_PORT="5000"
 ENV SERVER_HOST="0.0.0.0"
 ENV PYTHONUNBUFFERED=1
 # Set working directory
 WORKDIR /app
 # Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
    libgl1-mesa-glx \
    libglib2.0-0 \
    curl \
    && rm -rf /var/lib/apt/lists/*
 # Install Python dependencies (using stable paddleocr 2.x)
 RUN pip install --no-cache-dir \
    paddleocr==2.8.1 \
    fastapi \
    uvicorn[standard] \
    python-multipart \
    opencv-python-headless \
    pillow
 # Copy server files
 COPY image_support_files/paddleocr_server.py /app/paddleocr_server.py
 COPY image_support_files/paddleocr-entrypoint.sh /usr/local/bin/paddleocr-entrypoint.sh
 RUN chmod +x /usr/local/bin/paddleocr-entrypoint.sh
 # Note: OCR models will be downloaded on first run
 # This ensures compatibility across different GPU architectures
 # Expose API port
 EXPOSE 5000
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    CMD curl -f http://localhost:5000/health || exit 1
 ENTRYPOINT ["/usr/local/bin/paddleocr-entrypoint.sh"]
--- a/53
+++ b/53
@@ -1,53 +0,0 @@
 # PaddleOCR CPU Variant
 # OCR processing optimized for CPU-only inference
 FROM python:3.10-slim-bookworm
 LABEL maintainer="Task Venture Capital GmbH <hello@task.vc>"
 LABEL description="PaddleOCR PP-OCRv4 - CPU optimized"
 LABEL org.opencontainers.image.source="https://code.foss.global/host.today/ht-docker-ai"
 # Environment configuration for CPU-only mode
 ENV OCR_LANGUAGE="en"
 ENV SERVER_PORT="5000"
 ENV SERVER_HOST="0.0.0.0"
 ENV PYTHONUNBUFFERED=1
 # Disable GPU usage for CPU-only variant
 ENV CUDA_VISIBLE_DEVICES="-1"
 # Set working directory
 WORKDIR /app
 # Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
    libgl1-mesa-glx \
    libglib2.0-0 \
    libgomp1 \
    curl \
    && rm -rf /var/lib/apt/lists/*
 # Install Python dependencies (CPU version of PaddlePaddle - using stable 2.x versions)
 RUN pip install --no-cache-dir \
    paddlepaddle==2.6.2 \
    paddleocr==2.8.1 \
    fastapi \
    uvicorn[standard] \
    python-multipart \
    opencv-python-headless \
    pillow
 # Copy server files
 COPY image_support_files/paddleocr_server.py /app/paddleocr_server.py
 COPY image_support_files/paddleocr-entrypoint.sh /usr/local/bin/paddleocr-entrypoint.sh
 RUN chmod +x /usr/local/bin/paddleocr-entrypoint.sh
 # Note: OCR models will be downloaded on first run
 # This avoids build-time segfaults with certain CPU architectures
 # Expose API port
 EXPOSE 5000
 # Health check (longer start-period for CPU variant)
 HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
    CMD curl -f http://localhost:5000/health || exit 1
 ENTRYPOINT ["/usr/local/bin/paddleocr-entrypoint.sh"]
--- a/26
+++ b/26
@@ -0,0 +1,26 @@
 # Qwen3-VL-30B-A3B Vision Language Model
 # Q4_K_M quantization (~20GB model)
 #
 # Most powerful Qwen vision model:
 # - 256K context (expandable to 1M)
 # - Visual agent capabilities
 # - Code generation from images
 #
 # Build: docker build -f Dockerfile_qwen3vl -t qwen3vl .
 # Run:   docker run --gpus all -p 11434:11434 -v ht-ollama-models:/root/.ollama qwen3vl
 FROM ollama/ollama:latest
 # Pre-pull the model during build (optional - can also pull at runtime)
 # This makes the image larger but faster to start
 # RUN ollama serve & sleep 5 && ollama pull qwen3-vl:30b-a3b && pkill ollama
 # Expose Ollama API port
 EXPOSE 11434
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
  CMD curl -f http://localhost:11434/api/tags || exit 1
 # Start Ollama server
 CMD ["serve"]
--- a/build-images.sh
+++ b/build-images.sh
@@ -13,46 +13,38 @@ NC='\033[0m' # No Color
 echo -e "${BLUE}Building ht-docker-ai images...${NC}"
-# Build GPU variant
+# Build MiniCPM-V 4.5 GPU variant
 echo -e "${GREEN}Building MiniCPM-V 4.5 GPU variant...${NC}"
 docker build \
-    -f Dockerfile_minicpm45v \
+    -f Dockerfile_minicpm45v_ollama_gpu_VRAM9GB \
    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v \
    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v-gpu \
    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:latest \
    .
-# Build CPU variant
+# Build Qwen3-VL GPU variant
-echo -e "${GREEN}Building MiniCPM-V 4.5 CPU variant...${NC}"
+echo -e "${GREEN}Building Qwen3-VL-30B-A3B GPU variant...${NC}"
 docker build \
-    -f Dockerfile_minicpm45v_cpu \
+    -f Dockerfile_qwen3vl_ollama_gpu_VRAM20GB \
-    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v-cpu \
+    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:qwen3vl \
    .
-# Build PaddleOCR GPU variant
+# Build Nanonets-OCR GPU variant
-echo -e "${GREEN}Building PaddleOCR GPU variant...${NC}"
+echo -e "${GREEN}Building Nanonets-OCR-s GPU variant...${NC}"
 docker build \
-    -f Dockerfile_paddleocr \
+    -f Dockerfile_nanonets_vllm_gpu_VRAM10GB \
-    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr \
+    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:nanonets-ocr \
    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr-gpu \
    .
 # Build PaddleOCR CPU variant
 echo -e "${GREEN}Building PaddleOCR CPU variant...${NC}"
 docker build \
    -f Dockerfile_paddleocr_cpu \
    -t ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr-cpu \
    .
 echo -e "${GREEN}All images built successfully!${NC}"
 echo ""
 echo "Available images:"
-echo "  MiniCPM-V 4.5:"
+echo "  MiniCPM-V 4.5 (Ollama, ~9GB VRAM):"
-echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v (GPU)"
+echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v"
-echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:minicpm45v-cpu (CPU)"
+echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:latest"
 echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:latest (GPU)"
 echo ""
-echo "  PaddleOCR:"
+echo "  Qwen3-VL-30B-A3B (Ollama, ~20GB VRAM):"
-echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr (GPU)"
+echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:qwen3vl"
-echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr-gpu (GPU)"
+echo ""
-echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:paddleocr-cpu (CPU)"
+echo "  Nanonets-OCR-s (vLLM, ~10GB VRAM):"
 echo "  - ${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:nanonets-ocr"
--- a/changelog.md
+++ b/changelog.md
@@ -1,5 +1,177 @@
 # Changelog
 ## 2026-01-20 - 1.15.1 - fix(tests)
 enable progress events in invoice tests and bump @push.rocks/smartagent devDependency to ^1.5.4
 - Added an onProgress handler in test/test.invoices.nanonets.ts to log progress events (console.log(event.logMessage)) so tool calls and progress are visible during tests.
 - Bumped devDependency @push.rocks/smartagent from ^1.5.2 to ^1.5.4 in package.json.
 ## 2026-01-20 - 1.15.0 - feat(tests)
 integrate SmartAi/DualAgentOrchestrator into extraction tests and add JSON self-validation
 - Integrate SmartAi and DualAgentOrchestrator into bankstatement and invoice tests to perform structured extraction with streaming
 - Register and use JsonValidatorTool to validate outputs (json.validate) and enforce validation before task completion
 - Add tryExtractJson parsing fallback, improved extraction prompts, retries and clearer parsing/logging
 - Initialize and teardown SmartAi and orchestrator in test setup/summary, and enable onToken streaming handlers for real-time output
 - Bump devDependencies: @push.rocks/smartagent to ^1.3.0 and @push.rocks/smartai to ^0.12.0
 ## 2026-01-20 - 1.14.3 - fix(repo)
 no changes detected in the diff; no files modified and no release required
 - Diff contained no changes
 - No files were added, removed, or modified
 - No code, dependency, or documentation updates to release
 ## 2026-01-19 - 1.14.2 - fix(readme)
 update README to document Nanonets-OCR2-3B (replaces Nanonets-OCR-s), adjust VRAM and context defaults, expand feature docs, and update examples/test command
 - Renamed Nanonets-OCR-s -> Nanonets-OCR2-3B throughout README and examples
 - Updated Nanonets VRAM guidance from ~10GB to ~12-16GB and documented 30K context
 - Changed documented MAX_MODEL_LEN default from 8192 to 30000
 - Updated example model identifiers (model strings and curl/example snippets) to nanonets/Nanonets-OCR2-3B
 - Added MiniCPM and Qwen feature bullets (multilingual, multi-image, flowchart support, expanded context notes)
 - Replaced README test command from ./test-images.sh to pnpm test
 ## 2026-01-19 - 1.14.1 - fix(extraction)
 improve JSON extraction prompts and model options for invoice and bank statement tests
 - Refactor JSON extraction prompts to be sent after the document text and add explicit 'WHERE TO FIND DATA' and 'RULES' sections for clearer extraction guidance
 - Change chat message flow to: send document, assistant acknowledgement, then the JSON extraction prompt (avoids concatenating large prompts into one message)
 - Add model options (num_ctx: 32768, temperature: 0) to give larger context windows and deterministic JSON output
 - Simplify logging to avoid printing full prompt contents; log document and prompt lengths instead
 - Increase timeouts for large documents to 600000ms (10 minutes) where applicable
 ## 2026-01-19 - 1.14.0 - feat(docker-images)
 add vLLM-based Nanonets-OCR2-3B image, Qwen3-VL Ollama image and refactor build/docs/tests to use new runtime/layout
 - Add new Dockerfiles for Nanonets (Dockerfile_nanonets_vllm_gpu_VRAM10GB), Qwen3 (Dockerfile_qwen3vl_ollama_gpu_VRAM20GB) and a clarified MiniCPM Ollama variant (Dockerfile_minicpm45v_ollama_gpu_VRAM9GB); remove older, redundant Dockerfiles.
 - Update build-images.sh to build the new image tags (minicpm45v, qwen3vl, nanonets-ocr) and adjust messaging/targets accordingly.
 - Documentation overhaul: readme.md and readme.hints.md updated to reflect vLLM vs Ollama runtimes, corrected ports/VRAM estimates, volume recommendations, and API endpoint details.
 - Tests updated to target the new model ID (nanonets/Nanonets-OCR2-3B), to process one page per batch, and to include a 10-minute AbortSignal timeout for OCR requests.
 - Added focused extraction test suites (test/test.invoices.extraction.ts and test/test.invoices.failed.ts) for faster iteration and debugging of invoice extraction.
 - Bump devDependencies: @git.zone/tsrun -> ^2.0.1 and @git.zone/tstest -> ^3.1.5.
 - Misc: test helper references and docker compose/test port mapping fixed (nanonets uses 8000), and various README sections cleaned and reorganized.
 ## 2026-01-18 - 1.13.2 - fix(tests)
 stabilize OCR extraction tests and manage GPU containers
 - Add stopAllGpuContainers() and call it before starting GPU images to free GPU memory.
 - Remove PaddleOCR-VL image configs and associated ensure helpers from docker test helper to simplify images list.
 - Split invoice/bankstatement tests into two sequential stages: Stage 1 runs Nanonets OCR to produce markdown files, Stage 2 stops Nanonets and runs model extraction from saved markdown (avoids GPU contention).
 - Introduce temporary markdown directory handling and cleanup; add stopNanonets() and container running checks in tests.
 - Switch bank statement extraction model from qwen3:8b to gpt-oss:20b; add request timeout and improved logging/console output across tests.
 - Refactor extractWithConsensus and extraction functions to accept document identifiers, improve error messages and JSON extraction robustness.
 ## 2026-01-18 - 1.13.1 - fix(image_support_files)
 remove PaddleOCR-VL server scripts from image_support_files
 - Deleted files: image_support_files/paddleocr_vl_full_server.py (approx. 636 lines) and image_support_files/paddleocr_vl_server.py (approx. 465 lines)
 - Cleanup/removal of legacy PaddleOCR-VL FastAPI server implementations — may affect users who relied on these local scripts
 ## 2026-01-18 - 1.13.0 - feat(tests)
 revamp tests and remove legacy Dockerfiles: adopt JSON/consensus workflows, switch MiniCPM model, and delete deprecated Docker/test variants
 - Removed multiple Dockerfiles and related entrypoints for MiniCPM and PaddleOCR-VL (cpu/gpu/full), cleaning up legacy image recipes.
 - Pruned many older test files (combined, ministral3, paddleocr-vl, and several invoice/test variants) to consolidate the test suite.
 - Updated bank statement MiniCPM test: now uses MODEL='openbmb/minicpm-v4.5:q8_0', JSON per-page extraction prompt, consensus retry logic, expanded logging, and stricter result matching.
 - Updated invoice MiniCPM test: switched to a consensus flow (fast JSON pass + thinking pass), increased PDF conversion quality, endpoints migrated to chat-style API calls with image-in-message payloads, and improved finalization logic.
 - API usage changed from /api/generate to /api/chat with message-based payloads and embedded images — CI and local test runners will need model availability and possible pipeline adjustments.
 ## 2026-01-18 - 1.12.0 - feat(tests)
 switch vision tests to multi-query extraction (count then per-row/field queries) and add logging/summaries
 - Replace streaming + consensus pipeline with multi-query approach: count rows per page, then query each transaction/field individually (batched parallel queries).
 - Introduce unified helpers (queryVision / queryField / getTransaction / countTransactions) and simplify Ollama requests (stream:false, reduced num_predict, /no_think prompts).
 - Improve parsing and normalization for amounts (European formats), invoice numbers, dates and currency extraction.
 - Adjust model checks to look for generic 'minicpm' and update test names/messages; add pass/fail counters and a summary test output.
 - Remove previous consensus voting and streaming JSON accumulation logic, and add immediate per-transaction logging and batching.
 ## 2026-01-18 - 1.11.0 - feat(vision)
 process pages separately and make Qwen3-VL vision extraction more robust; add per-page parsing, safer JSON handling, reduced token usage, and multi-query invoice extraction
 - Bank statements: split extraction into extractTransactionsFromPage and sequentially process pages to avoid thinking-token exhaustion
 - Bank statements: reduced num_predict from 8000 to 4000, send single image per request, added per-page logging and non-throwing handling for empty or non-JSON responses
 - Bank statements: catch JSON.parse errors and return empty array instead of throwing
 - Invoices: introduced queryField to request single values and perform multiple simple queries (reduces model thinking usage)
 - Invoices: reduced num_predict for invoice queries from 4000 to 500 and parse amounts robustly (handles European formats like 1.234,56)
 - Invoices: normalize currency to uppercase 3-letter code, return safe defaults (empty strings / 0) instead of nulls, and parse net/vat/total with fallbacks
 - General: simplified Ollama API error messages to avoid including response body content in thrown errors
 ## 2026-01-18 - 1.10.1 - fix(tests)
 improve Qwen3-VL invoice extraction test by switching to non-stream API, adding model availability/pull checks, simplifying response parsing, and tightening model options
 - Replaced streaming reader logic with direct JSON parsing of the /api/chat response
 - Added ensureQwen3Vl() to check and pull the Qwen3-VL:8b model from Ollama
 - Switched to ensureMiniCpm() to verify Ollama service is running before model checks
 - Use /no_think prompt for direct JSON output and set temperature to 0.0 and num_predict to 512
 - Removed retry loop and streaming parsing; improved error messages to include response body
 - Updated logging and test setup messages for clarity
 ## 2026-01-18 - 1.10.0 - feat(vision)
 add Qwen3-VL vision model support with Dockerfile and tests; improve invoice OCR conversion and prompts; simplify extraction flow by removing consensus voting
 - Add Dockerfile_qwen3vl to provide an Ollama-based image for Qwen3-VL and expose the Ollama API on port 11434
 - Introduce test/test.invoices.qwen3vl.ts and ensureQwen3Vl() helper to pull and test qwen3-vl:8b
 - Improve PDF->PNG conversion and prompt in ministral3 tests (higher DPI, max quality, sharpen) and increase num_predict from 512 to 1024
 - Simplify extraction pipeline: remove consensus voting, log single-pass results, and simplify OCR HTML sanitization/truncation logic
 ## 2026-01-18 - 1.9.0 - feat(tests)
 add Ministral 3 vision tests and improve invoice extraction pipeline to use Ollama chat schema, sanitization, and multi-page support
 - Add new vision-based test suites for Ministral 3: test/test.invoices.ministral3.ts and test/test.bankstatements.ministral3.ts (model ministral-3:8b).
 - Introduce ensureMinistral3() helper to start/check Ollama/MiniCPM model in test/helpers/docker.ts.
 - Switch invoice extraction to use Ollama /api/chat with a JSON schema (format) and streaming support (reads message.content).
 - Improve HTML handling: sanitizeHtml() to remove OCR artifacts, concatenate multi-page HTML with page markers, and increase truncation limits.
 - Enhance response parsing: strip Markdown code fences, robustly locate JSON object boundaries, and provide clearer JSON parse errors.
 - Add PDF->PNG conversion (ImageMagick) and direct image-based extraction flow for vision model tests.
 ## 2026-01-18 - 1.8.0 - feat(paddleocr-vl)
 add structured HTML output and table parsing for PaddleOCR-VL, update API, tests, and README
 - Add result_to_html(), parse_markdown_table(), and parse_paddleocr_table() to emit semantic HTML and convert OCR/markdown tables to proper <table> elements
 - Enhance result_to_markdown() with positional/type hints (header/footer/title/table/figure) to improve downstream LLM processing
 - Expose 'html' in supported formats and handle output_format='html' in parse endpoints and CLI flow
 - Update tests to request HTML output and extract invoice fields from structured HTML (test/test.invoices.paddleocr-vl.ts)
 - Refresh README with usage, new images/tags, architecture notes, and troubleshooting for the updated pipeline
 ## 2026-01-17 - 1.7.1 - fix(docker)
 standardize Dockerfile and entrypoint filenames; add GPU-specific Dockerfiles and update build and test references
 - Added Dockerfile_minicpm45v_gpu and image_support_files/minicpm45v_entrypoint.sh; removed the old Dockerfile_minicpm45v and docker-entrypoint.sh
 - Renamed and simplified PaddleOCR entrypoint to image_support_files/paddleocr_vl_entrypoint.sh and updated CPU/GPU Dockerfile references
 - Updated build-images.sh to use *_gpu Dockerfiles and clarified PaddleOCR GPU build log
 - Updated test/helpers/docker.ts to point to Dockerfile_minicpm45v_gpu so tests build the GPU variant
 ## 2026-01-17 - 1.7.0 - feat(tests)
 use Qwen2.5 (Ollama) for invoice extraction tests and add helpers for model management; normalize dates and coerce numeric fields
 - Added ensureOllamaModel and ensureQwen25 test helpers to pull/check Ollama models via localhost:11434
 - Updated invoices test to use qwen2.5:7b instead of MiniCPM and removed image payload from the text-only extraction step
 - Increased Markdown truncate limit from 8000 to 12000 and reduced model num_predict from 2048 to 512
 - Rewrote extraction prompt to require strict JSON output and added post-processing to parse/convert numeric fields
 - Added normalizeDate and improved compareInvoice to normalize dates and handle numeric formatting/tolerance
 - Updated test setup to ensure Qwen2.5 is available and adjusted logging/messages to reflect the Qwen2.5-based workflow
 ## 2026-01-17 - 1.6.0 - feat(paddleocr-vl)
 add PaddleOCR-VL full pipeline Docker image and API server, plus integration tests and docker helpers
 - Add Dockerfile_paddleocr_vl_full and entrypoint script to build a GPU-enabled image with PP-DocLayoutV2 + PaddleOCR-VL and a FastAPI server
 - Introduce image_support_files/paddleocr_vl_full_server.py implementing the full pipeline API (/parse, OpenAI-compatible /v1/chat/completions) and a /formats endpoint
 - Improve image handling: decode_image supports data URLs, HTTP(S), raw base64 and file paths; add optimize_image_resolution to auto-scale images into the recommended 1080-2048px range
 - Add test helpers (test/helpers/docker.ts) to build/start/health-check Docker images and new ensurePaddleOcrVlFull workflow
 - Add comprehensive integration tests for bank statements and invoices (MiniCPM and PaddleOCR-VL variants) and update tests to ensure required containers are running before tests
 - Switch MiniCPM model references to 'minicpm-v:latest' and increase health/timeout expectations for the full pipeline
 ## 2026-01-17 - 1.5.0 - feat(paddleocr-vl)
 add PaddleOCR-VL GPU Dockerfile, pin vllm, update CPU image deps, and improve entrypoint and tests
 - Add a new GPU Dockerfile for PaddleOCR-VL (transformers-based) with CUDA support, healthcheck, and entrypoint.
 - Pin vllm to 0.11.1 in Dockerfile_paddleocr_vl to use the first stable release with PaddleOCR-VL support.
 - Update CPU image: add torchvision==0.20.1 and extra Python deps (protobuf, sentencepiece, einops) required by the transformers-based server.
 - Rewrite paddleocr-vl-entrypoint.sh to build vllm args array, add MAX_MODEL_LEN and ENFORCE_EAGER env vars, include --limit-mm-per-prompt and optional --enforce-eager, and switch to exec vllm with constructed args.
 - Update tests to use the OpenAI-compatible PaddleOCR-VL chat completions API (/v1/chat/completions) with image+text message payload and model 'paddleocr-vl'.
 - Add @types/node to package.json dependencies and tidy devDependencies ordering.
 ## 2026-01-16 - 1.4.0 - feat(invoices)
 add hybrid OCR + vision invoice/document parsing with PaddleOCR, consensus voting, and prompt/test refactors
--- a/image_support_files/minicpm45v_entrypoint.sh
+++ b/image_support_files/minicpm45v_entrypoint.sh
--- a/image_support_files/paddleocr-entrypoint.sh
+++ b/image_support_files/paddleocr-entrypoint.sh
@@ -1,25 +0,0 @@
 #!/bin/bash
 set -e
 # Configuration from environment
 OCR_LANGUAGE="${OCR_LANGUAGE:-en}"
 SERVER_PORT="${SERVER_PORT:-5000}"
 SERVER_HOST="${SERVER_HOST:-0.0.0.0}"
 echo "Starting PaddleOCR Server..."
 echo "  Language: ${OCR_LANGUAGE}"
 echo "  Host: ${SERVER_HOST}"
 echo "  Port: ${SERVER_PORT}"
 # Check GPU availability
 if [ "${CUDA_VISIBLE_DEVICES}" = "-1" ]; then
    echo "  GPU: Disabled (CPU mode)"
 else
    echo "  GPU: Enabled"
 fi
 # Start the FastAPI server with uvicorn
 exec python -m uvicorn paddleocr_server:app \
    --host "${SERVER_HOST}" \
    --port "${SERVER_PORT}" \
    --workers 1
--- a/image_support_files/paddleocr_server.py
+++ b/image_support_files/paddleocr_server.py
@@ -1,253 +0,0 @@
 #!/usr/bin/env python3
 """
 PaddleOCR FastAPI Server
 Provides REST API for OCR operations using PaddleOCR
 """
 import os
 import io
 import base64
 import logging
 from typing import Optional, List, Any
 from fastapi import FastAPI, File, UploadFile, Form, HTTPException
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 import numpy as np
 from PIL import Image
 from paddleocr import PaddleOCR
 # Configure logging
 logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
 )
 logger = logging.getLogger(__name__)
 # Environment configuration
 OCR_LANGUAGE = os.environ.get('OCR_LANGUAGE', 'en')
 # GPU is controlled via CUDA_VISIBLE_DEVICES environment variable
 USE_GPU = os.environ.get('CUDA_VISIBLE_DEVICES', '') != '-1'
 # Initialize FastAPI app
 app = FastAPI(
    title="PaddleOCR Server",
    description="REST API for OCR operations using PaddleOCR PP-OCRv4",
    version="1.0.0"
 )
 # Global OCR instance
 ocr_instance: Optional[PaddleOCR] = None
 class OCRRequest(BaseModel):
    """Request model for base64 image OCR"""
    image: str
    language: Optional[str] = None
 class BoundingBox(BaseModel):
    """Bounding box for detected text"""
    points: List[List[float]]
 class OCRResult(BaseModel):
    """Single OCR detection result"""
    text: str
    confidence: float
    box: List[List[float]]
 class OCRResponse(BaseModel):
    """OCR response model"""
    success: bool
    results: List[OCRResult]
    error: Optional[str] = None
 class HealthResponse(BaseModel):
    """Health check response"""
    status: str
    model: str
    language: str
    gpu_enabled: bool
 def get_ocr(lang: Optional[str] = None) -> PaddleOCR:
    """Get or initialize the OCR instance"""
    global ocr_instance
    use_lang = lang or OCR_LANGUAGE
    # Return cached instance if same language
    if ocr_instance is not None and lang is None:
        return ocr_instance
    logger.info(f"Initializing PaddleOCR with language={use_lang}, use_gpu={USE_GPU}")
    new_ocr = PaddleOCR(
        use_angle_cls=True,
        lang=use_lang,
        use_gpu=USE_GPU,
        show_log=False
    )
    # Cache the default language instance
    if lang is None:
        ocr_instance = new_ocr
    logger.info("PaddleOCR initialized successfully")
    return new_ocr
 def decode_base64_image(base64_string: str) -> np.ndarray:
    """Decode base64 string to numpy array"""
    # Remove data URL prefix if present
    if ',' in base64_string:
        base64_string = base64_string.split(',')[1]
    image_data = base64.b64decode(base64_string)
    image = Image.open(io.BytesIO(image_data))
    # Convert to RGB if necessary
    if image.mode != 'RGB':
        image = image.convert('RGB')
    return np.array(image)
 def process_ocr_result(result: Any) -> List[OCRResult]:
    """Process PaddleOCR result into structured format"""
    results = []
    if result is None or len(result) == 0:
        return results
    # PaddleOCR returns list of results per image
    # Each result is a list of [box, (text, confidence)]
    for line in result[0] if result[0] else []:
        if line is None:
            continue
        box = line[0]  # [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
        text_info = line[1]  # (text, confidence)
        results.append(OCRResult(
            text=text_info[0],
            confidence=float(text_info[1]),
            box=[[float(p[0]), float(p[1])] for p in box]
        ))
    return results
@app.on_event("startup")
 async def startup_event():
    """Pre-warm the OCR model on startup"""
    logger.info("Pre-warming OCR model...")
    try:
        ocr = get_ocr()
        # Create a small test image to warm up the model
        test_image = np.zeros((100, 100, 3), dtype=np.uint8)
        test_image.fill(255)  # White image
        ocr.ocr(test_image, cls=True)
        logger.info("OCR model pre-warmed successfully")
    except Exception as e:
        logger.error(f"Failed to pre-warm OCR model: {e}")
@app.get("/health", response_model=HealthResponse)
 async def health_check():
    """Health check endpoint"""
    try:
        # Ensure OCR is initialized
        get_ocr()
        return HealthResponse(
            status="healthy",
            model="PP-OCRv4",
            language=OCR_LANGUAGE,
            gpu_enabled=USE_GPU
        )
    except Exception as e:
        logger.error(f"Health check failed: {e}")
        raise HTTPException(status_code=503, detail=str(e))
@app.post("/ocr", response_model=OCRResponse)
 async def ocr_base64(request: OCRRequest):
    """
    Perform OCR on a base64-encoded image
    Args:
        request: OCRRequest with base64 image and optional language
    Returns:
        OCRResponse with detected text, confidence scores, and bounding boxes
    """
    try:
        # Decode image
        image = decode_base64_image(request.image)
        # Get OCR instance (use request language if provided)
        if request.language and request.language != OCR_LANGUAGE:
            ocr = get_ocr(request.language)
        else:
            ocr = get_ocr()
        result = ocr.ocr(image, cls=True)
        # Process results
        results = process_ocr_result(result)
        return OCRResponse(success=True, results=results)
    except Exception as e:
        logger.error(f"OCR processing failed: {e}")
        return OCRResponse(success=False, results=[], error=str(e))
@app.post("/ocr/upload", response_model=OCRResponse)
 async def ocr_upload(
    img: UploadFile = File(...),
    language: Optional[str] = Form(None)
 ):
    """
    Perform OCR on an uploaded image file
    Args:
        img: Uploaded image file
        language: Optional language code (default: env OCR_LANGUAGE)
    Returns:
        OCRResponse with detected text, confidence scores, and bounding boxes
    """
    try:
        # Read image
        contents = await img.read()
        image = Image.open(io.BytesIO(contents))
        # Convert to RGB if necessary
        if image.mode != 'RGB':
            image = image.convert('RGB')
        image_array = np.array(image)
        # Get OCR instance
        if language and language != OCR_LANGUAGE:
            ocr = get_ocr(language)
        else:
            ocr = get_ocr()
        result = ocr.ocr(image_array, cls=True)
        # Process results
        results = process_ocr_result(result)
        return OCRResponse(success=True, results=results)
    except Exception as e:
        logger.error(f"OCR processing failed: {e}")
        return OCRResponse(success=False, results=[], error=str(e))
 if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=5000)
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@host.today/ht-docker-ai",
-  "version": "1.4.0",
+  "version": "1.15.1",
  "type": "module",
  "private": false,
  "description": "Docker images for AI vision-language models including MiniCPM-V 4.5",
@@ -13,8 +13,10 @@
    "test": "tstest test/ --verbose"
  },
  "devDependencies": {
-    "@git.zone/tstest": "^1.0.90",
+    "@git.zone/tsrun": "^2.0.1",
-    "@git.zone/tsrun": "^1.3.3"
+    "@git.zone/tstest": "^3.1.5",
    "@push.rocks/smartagent": "^1.5.4",
    "@push.rocks/smartai": "^0.13.2"
  },
  "repository": {
    "type": "git",
@@ -28,5 +30,8 @@
    "minicpm",
    "ollama",
    "multimodal"
-  ]
+  ],
  "dependencies": {
    "@types/node": "^25.0.9"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/readme.hints.md
+++ b/readme.hints.md
@@ -2,12 +2,18 @@
 ## Architecture
-This project uses **Ollama** as the runtime framework for serving AI models. This provides:
+This project uses **Ollama** and **vLLM** as runtime frameworks for serving AI models:
 ### Ollama-based Images (MiniCPM-V, Qwen3-VL)
 - Automatic model download and caching
 - Unified REST API (compatible with OpenAI format)
 - Built-in quantization support
- GPU/CPU auto-detection
+- GPU auto-detection
 ### vLLM-based Images (Nanonets-OCR)
 - High-performance inference server
 - OpenAI-compatible API
 - Optimized for VLM workloads
 ## Model Details
@@ -24,18 +30,24 @@ This project uses **Ollama** as the runtime framework for serving AI models. Thi
 |------|---------------|
 | Full precision (bf16) | 18GB |
 | int4 quantized | 9GB |
 | GGUF (CPU) | 8GB RAM |
 ## Container Startup Flow
 ### Ollama-based containers
 1. `docker-entrypoint.sh` starts Ollama server in background
 2. Waits for server to be ready
 3. Checks if model already exists in volume
 4. Pulls model if not present
 5. Keeps container running
 ### vLLM-based containers
 1. vLLM server starts with model auto-download
 2. Health check endpoint available at `/health`
 3. OpenAI-compatible API at `/v1/chat/completions`
 ## Volume Persistence
 ### Ollama volumes
 Mount `/root/.ollama` to persist downloaded models:
 ```bash
@@ -44,9 +56,16 @@ Mount `/root/.ollama` to persist downloaded models:
 Without this volume, the model will be re-downloaded on each container start (~5GB download).
 ### vLLM/HuggingFace volumes
 Mount `/root/.cache/huggingface` for model caching:
 ```bash
 -v hf-cache:/root/.cache/huggingface
 ```
 ## API Endpoints
-All endpoints follow the Ollama API specification:
+### Ollama API (MiniCPM-V, Qwen3-VL)
 | Endpoint | Method | Description |
 |----------|--------|-------------|
@@ -56,99 +75,137 @@ All endpoints follow the Ollama API specification:
 | `/api/pull` | POST | Pull a model |
 | `/api/show` | POST | Show model info |
-## GPU Detection
+### vLLM API (Nanonets-OCR)
-The GPU variant uses Ollama's automatic GPU detection. For CPU-only mode, we set:
+| Endpoint | Method | Description |
-
+|----------|--------|-------------|
-```dockerfile
+| `/health` | GET | Health check |
-ENV CUDA_VISIBLE_DEVICES=""
+| `/v1/models` | GET | List available models |
-```
+| `/v1/chat/completions` | POST | OpenAI-compatible chat completions |
 This forces Ollama to use CPU inference even if GPU is available.
 ## Health Checks
-Both variants include Docker health checks:
+All containers include Docker health checks:
 ```dockerfile
 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
    CMD curl -f http://localhost:11434/api/tags || exit 1
 ```
-CPU variant has longer `start-period` (120s) due to slower startup.
+---
-## PaddleOCR
+## Nanonets-OCR-s
 ### Overview
-PaddleOCR is a standalone OCR service using PaddlePaddle's PP-OCRv4 model. It provides:
+Nanonets-OCR-s is a Qwen2.5-VL-3B model fine-tuned specifically for document OCR tasks. It outputs structured markdown with semantic tags.
- Text detection and recognition
+**Key features:**
- Multi-language support
+- Based on Qwen2.5-VL-3B (~4B parameters)
- FastAPI REST API
+- Fine-tuned for document OCR
- GPU and CPU variants
+- Outputs markdown with semantic HTML tags
 - ~10GB VRAM
 ### Docker Images
 | Tag | Description |
 |-----|-------------|
-| `paddleocr` | GPU variant (default) |
+| `nanonets-ocr` | GPU variant using vLLM (OpenAI-compatible API) |
 | `paddleocr-gpu` | GPU variant (alias) |
 | `paddleocr-cpu` | CPU-only variant |
-### API Endpoints
+### API Endpoints (OpenAI-compatible via vLLM)
 | Endpoint | Method | Description |
 |----------|--------|-------------|
-| `/health` | GET | Health check with model info |
+| `/health` | GET | Health check |
-| `/ocr` | POST | OCR with base64 image (JSON body) |
+| `/v1/models` | GET | List available models |
-| `/ocr/upload` | POST | OCR with file upload (multipart form) |
+| `/v1/chat/completions` | POST | OpenAI-compatible chat completions |
 ### Request/Response Format
-**POST /ocr (JSON)**
+**POST /v1/chat/completions (OpenAI-compatible)**
 ```json
 {
-  "image": "<base64-encoded-image>",
+  "model": "nanonets/Nanonets-OCR-s",
-  "language": "en"  // optional
+  "messages": [
 }
 ```
 **POST /ocr/upload (multipart)**
 - `img`: image file
 - `language`: optional language code
 **Response**
 ```json
 {
  "success": true,
  "results": [
    {
-      "text": "Invoice #12345",
+      "role": "user",
-      "confidence": 0.98,
+      "content": [
-      "box": [[x1,y1], [x2,y2], [x3,y3], [x4,y4]]
+        {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}},
        {"type": "text", "text": "Extract the text from the above document..."}
      ]
    }
-  ]
+  ],
  "temperature": 0.0,
  "max_tokens": 4096
 }
 ```
-### Environment Variables
+### Nanonets OCR Prompt
-| Variable | Default | Description |
+The model is designed to work with a specific prompt format:
-|----------|---------|-------------|
+```
-| `OCR_LANGUAGE` | `en` | Default language for OCR |
+Extract the text from the above document as if you were reading it naturally.
-| `SERVER_PORT` | `5000` | Server port |
+Return the tables in html format.
-| `SERVER_HOST` | `0.0.0.0` | Server host |
+Return the equations in LaTeX representation.
-| `CUDA_VISIBLE_DEVICES` | (auto) | Set to `-1` for CPU-only |
+If there is an image in the document and image caption is not present, add a small description inside <img></img> tag.
 Watermarks should be wrapped in brackets. Ex: <watermark>OFFICIAL COPY</watermark>.
 Page numbers should be wrapped in brackets. Ex: <page_number>14</page_number>.
 ```
 ### Performance
- **GPU**: ~1-3 seconds per page
+- **GPU (vLLM)**: ~3-8 seconds per page
- **CPU**: ~10-30 seconds per page
+- **VRAM usage**: ~10GB
-### Supported Languages
+### Two-Stage Pipeline (Nanonets + Qwen3)
-Common language codes: `en` (English), `ch` (Chinese), `de` (German), `fr` (French), `es` (Spanish), `ja` (Japanese), `ko` (Korean)
+The Nanonets tests use a two-stage pipeline:
 1. **Stage 1**: Nanonets-OCR-s converts images to markdown (via vLLM on port 8000)
 2. **Stage 2**: Qwen3 8B extracts structured JSON from markdown (via Ollama on port 11434)
 **GPU Limitation**: Both vLLM and Ollama require significant GPU memory. On a single GPU system:
 - Running both simultaneously causes memory contention
 - For single GPU: Run services sequentially (stop Nanonets before Qwen3)
 - For multi-GPU: Assign each service to a different GPU
 **Sequential Execution**:
 ```bash
 # Step 1: Run Nanonets OCR (converts to markdown)
 docker start nanonets-test
 # ... perform OCR ...
 docker stop nanonets-test
 # Step 2: Run Qwen3 extraction (from markdown)
 docker start minicpm-test
 # ... extract JSON ...
 ```
 ---
 ## Multi-Pass Extraction Strategy
 The bank statement extraction uses a dual-VLM consensus approach:
 ### Architecture: Dual-VLM Consensus
 | VLM | Model | Purpose |
 |-----|-------|---------|
 | **MiniCPM-V 4.5** | 8B params | Primary visual extraction |
 | **Nanonets-OCR-s** | ~4B params | Document OCR with semantic output |
 ### Extraction Strategy
 1. **Pass 1**: MiniCPM-V visual extraction (images → JSON)
 2. **Pass 2**: Nanonets-OCR semantic extraction (images → markdown → JSON)
 3. **Consensus**: If Pass 1 == Pass 2 → Done (fast path)
 4. **Pass 3+**: MiniCPM-V visual if no consensus
 ### Why Dual-VLM Works
 - **Different architectures**: Two independent models cross-check each other
 - **Specialized strengths**: Nanonets-OCR-s optimized for document structure, MiniCPM-V for general vision
 - **No structure loss**: Both VLMs see the original images directly
 - **Fast consensus**: Most documents complete in 2 passes when VLMs agree
 ---
@@ -156,7 +213,7 @@ Common language codes: `en` (English), `ch` (Chinese), `de` (German), `fr` (Fren
 To add a new model variant:
-1. Create `Dockerfile_<modelname>`
+1. Create `Dockerfile_<modelname>_<runtime>_<hardware>_VRAM<size>`
 2. Set `MODEL_NAME` environment variable
 3. Update `build-images.sh` with new build target
 4. Add documentation to `readme.md`
@@ -174,8 +231,8 @@ The model download is ~5GB and may take several minutes.
 ### Out of memory
- GPU: Use int4 quantized version or add more VRAM
+- GPU: Use a lighter model variant or upgrade VRAM
- CPU: Increase container memory limit: `--memory=16g`
+- Add more GPU memory: Consider multi-GPU setup
 ### API not responding
@@ -193,8 +250,11 @@ npmci docker build
 npmci docker push code.foss.global
 ```
 ---
 ## Related Resources
 - [Ollama Documentation](https://ollama.ai/docs)
 - [MiniCPM-V GitHub](https://github.com/OpenBMB/MiniCPM-V)
 - [Ollama API Reference](https://github.com/ollama/ollama/blob/main/docs/api.md)
 - [Nanonets-OCR-s on HuggingFace](https://huggingface.co/nanonets/Nanonets-OCR-s)
--- a/readme.md
+++ b/readme.md
@@ -1,22 +1,51 @@
-# @host.today/ht-docker-ai
+# @host.today/ht-docker-ai 🚀
-Docker images for AI vision-language models, starting with MiniCPM-V 4.5.
+Production-ready Docker images for state-of-the-art AI Vision-Language Models. Run powerful multimodal AI locally with GPU acceleration—**no cloud API keys required**.
-## Overview
+> 🔥 **Three VLMs, one registry.** From high-performance document OCR to GPT-4o-level vision understanding—pick the right tool for your task.
-This project provides ready-to-use Docker containers for running state-of-the-art AI vision-language models. Built on Ollama for simplified model management and a consistent REST API.
+## Issue Reporting and Security
-## Available Images
+For reporting bugs, issues, or security vulnerabilities, please visit [community.foss.global/](https://community.foss.global/). This is the central community hub for all issue reporting. Developers who sign and comply with our contribution agreement and go through identification can also get a [code.foss.global/](https://code.foss.global/) account to submit Pull Requests directly.
-| Tag | Description | Requirements |
+---
 |-----|-------------|--------------|
 | `minicpm45v` | MiniCPM-V 4.5 with GPU support | NVIDIA GPU, 9-18GB VRAM |
 | `minicpm45v-cpu` | MiniCPM-V 4.5 CPU-only | 8GB+ RAM |
 | `latest` | Alias for `minicpm45v` | NVIDIA GPU |
-## Quick Start
+## 🎯 What's Included
-### GPU (Recommended)
+| Model | Parameters | Best For | API | Port | VRAM |
 |-------|-----------|----------|-----|------|------|
 | **MiniCPM-V 4.5** | 8B | General vision understanding, multi-image analysis | Ollama-compatible | 11434 | ~9GB |
 | **Nanonets-OCR2-3B** | ~3B | Document OCR with semantic markdown, LaTeX, flowcharts | OpenAI-compatible | 8000 | ~12-16GB |
 | **Qwen3-VL-30B** | 30B (A3B) | Advanced visual agents, code generation from images | Ollama-compatible | 11434 | ~20GB |
 ---
 ## 📦 Quick Reference: All Available Images
 ```
 code.foss.global/host.today/ht-docker-ai:<tag>
 ```
 | Tag | Model | Runtime | Port | VRAM |
 |-----|-------|---------|------|------|
 | `minicpm45v` / `latest` | MiniCPM-V 4.5 | Ollama | 11434 | ~9GB |
 | `nanonets-ocr` | Nanonets-OCR2-3B | vLLM | 8000 | ~12-16GB |
 | `qwen3vl` | Qwen3-VL-30B-A3B | Ollama | 11434 | ~20GB |
 ---
 ## 🖼️ MiniCPM-V 4.5
 A GPT-4o level multimodal LLM from OpenBMB—handles image understanding, OCR, multi-image analysis, and visual reasoning across **30+ languages**.
 ### ✨ Key Features
 - 🌍 **Multilingual:** 30+ languages supported
 - 🖼️ **Multi-image:** Analyze multiple images in one request
 - 📊 **Versatile:** Charts, documents, photos, diagrams
 - ⚡ **Efficient:** Runs on consumer GPUs (9GB VRAM)
 ### Quick Start
 ```bash
 docker run -d \
@@ -27,28 +56,16 @@ docker run -d \
  code.foss.global/host.today/ht-docker-ai:minicpm45v
 ```
-### CPU Only
+> 💡 **Pro tip:** Mount the volume to persist downloaded models (~5GB). Without it, models re-download on every container start.
-```bash
+### API Examples
 docker run -d \
  --name minicpm \
  -p 11434:11434 \
  -v ollama-data:/root/.ollama \
  code.foss.global/host.today/ht-docker-ai:minicpm45v-cpu
 ```
 ## API Usage
 The container exposes the Ollama API on port 11434.
 ### List Available Models
 **List models:**
 ```bash
 curl http://localhost:11434/api/tags
 ```
-### Generate Text from Image
+**Analyze an image:**
 ```bash
 curl http://localhost:11434/api/generate -d '{
  "model": "minicpm-v",
@@ -57,60 +74,149 @@ curl http://localhost:11434/api/generate -d '{
 }'
 ```
-### Chat with Vision
+**Chat with vision:**
 ```bash
 curl http://localhost:11434/api/chat -d '{
  "model": "minicpm-v",
-  "messages": [
+  "messages": [{
-    {
+    "role": "user",
-      "role": "user",
+    "content": "Describe this image in detail",
-      "content": "Describe this image in detail",
+    "images": ["<base64-encoded-image>"]
-      "images": ["<base64-encoded-image>"]
+  }]
    }
  ]
 }'
 ```
-## Environment Variables
+### Hardware Requirements
-| Variable | Default | Description |
+| Mode | VRAM Required |
-|----------|---------|-------------|
+|------|---------------|
-| `MODEL_NAME` | `minicpm-v` | Model to pull on startup |
+| int4 quantized | ~9GB |
-| `OLLAMA_HOST` | `0.0.0.0` | Host address for API |
+| Full precision (bf16) | ~18GB |
 | `OLLAMA_ORIGINS` | `*` | Allowed CORS origins |
-## Hardware Requirements
+---
-### GPU Variant (`minicpm45v`)
+## 🔍 Nanonets-OCR2-3B
- NVIDIA GPU with CUDA support
+The **latest Nanonets document OCR model** (October 2025 release)—based on Qwen2.5-VL-3B, fine-tuned specifically for document extraction with significant improvements over the original OCR-s.
 - Minimum 9GB VRAM (int4 quantized)
 - Recommended 18GB VRAM (full precision)
 - NVIDIA Container Toolkit installed
-### CPU Variant (`minicpm45v-cpu`)
+### ✨ Key Features
- Minimum 8GB RAM
+- 📝 **Semantic output:** Tables → HTML, equations → LaTeX, flowcharts → structured markup
- Recommended 16GB+ RAM for better performance
+- 🌍 **Multilingual:** Inherits Qwen's broad language support
- No GPU required
+- 📄 **30K context:** Handle large, multi-page documents
 - 🔌 **OpenAI-compatible:** Drop-in replacement for existing pipelines
 - 🎯 **Improved accuracy:** Better semantic tagging and LaTeX equation extraction vs. OCR-s
-## Model Information
+### Quick Start
-**MiniCPM-V 4.5** is a GPT-4o level multimodal large language model developed by OpenBMB.
+```bash
 docker run -d \
  --name nanonets \
  --gpus all \
  -p 8000:8000 \
  -v hf-cache:/root/.cache/huggingface \
  code.foss.global/host.today/ht-docker-ai:nanonets-ocr
 ```
- **Parameters**: 8B (Qwen3-8B + SigLIP2-400M)
+### API Usage
 - **Capabilities**: Image understanding, OCR, multi-image analysis
 - **Languages**: 30+ languages including English, Chinese, French, Spanish
-## Docker Compose Example
+```bash
 curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "nanonets/Nanonets-OCR2-3B",
    "messages": [{
      "role": "user",
      "content": [
        {"type": "image_url", "image_url": {"url": "data:image/png;base64,<base64>"}},
        {"type": "text", "text": "Extract the text from the above document as if you were reading it naturally. Return the tables in html format. Return the equations in LaTeX representation."}
      ]
    }],
    "temperature": 0.0,
    "max_tokens": 4096
  }'
 ```
 ### Output Format
 Nanonets-OCR2-3B returns markdown with semantic tags:
 | Element | Output Format |
 |---------|---------------|
 | Tables | `<table>...</table>` (HTML) |
 | Equations | `$...$` (LaTeX) |
 | Images | `<img>description</img>` |
 | Watermarks | `<watermark>OFFICIAL COPY</watermark>` |
 | Page numbers | `<page_number>14</page_number>` |
 | Flowcharts | Structured markup |
 ### Hardware Requirements
 | Config | VRAM |
 |--------|------|
 | 30K context (default) | ~12-16GB |
 | Speed | ~3-8 seconds per page |
 ---
 ## 🧠 Qwen3-VL-30B-A3B
 The **most powerful** Qwen vision model—30B parameters with 3B active (MoE architecture). Handles complex visual reasoning, code generation from screenshots, and visual agent capabilities.
 ### ✨ Key Features
 - 🚀 **256K context** (expandable to 1M tokens!)
 - 🤖 **Visual agent capabilities** — can plan and execute multi-step tasks
 - 💻 **Code generation from images** — screenshot → working code
 - 🎯 **State-of-the-art** visual reasoning
 ### Quick Start
 ```bash
 docker run -d \
  --name qwen3vl \
  --gpus all \
  -p 11434:11434 \
  -v ollama-data:/root/.ollama \
  code.foss.global/host.today/ht-docker-ai:qwen3vl
 ```
 Then pull the model (one-time, ~20GB):
 ```bash
 docker exec qwen3vl ollama pull qwen3-vl:30b-a3b
 ```
 ### API Usage
 ```bash
 curl http://localhost:11434/api/chat -d '{
  "model": "qwen3-vl:30b-a3b",
  "messages": [{
    "role": "user",
    "content": "Analyze this screenshot and write the code to recreate this UI",
    "images": ["<base64-encoded-image>"]
  }]
 }'
 ```
 ### Hardware Requirements
 | Requirement | Value |
 |-------------|-------|
 | VRAM | ~20GB (Q4_K_M quantization) |
 | Context | 256K tokens default |
 ---
 ## 🐳 Docker Compose
 Run multiple VLMs together for maximum flexibility:
 ```yaml
 version: '3.8'
 services:
  # General vision tasks
  minicpm:
    image: code.foss.global/host.today/ht-docker-ai:minicpm45v
    container_name: minicpm
    ports:
      - "11434:11434"
    volumes:
@@ -124,11 +230,84 @@ services:
              capabilities: [gpu]
    restart: unless-stopped
  # Document OCR with semantic output
  nanonets:
    image: code.foss.global/host.today/ht-docker-ai:nanonets-ocr
    ports:
      - "8000:8000"
    volumes:
      - hf-cache:/root/.cache/huggingface
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    restart: unless-stopped
 volumes:
  ollama-data:
  hf-cache:
 ```
-## Building Locally
+---
 ## ⚙️ Environment Variables
 ### MiniCPM-V 4.5 & Qwen3-VL (Ollama-based)
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `MODEL_NAME` | `minicpm-v` | Ollama model to pull on startup |
 | `OLLAMA_HOST` | `0.0.0.0` | API bind address |
 | `OLLAMA_ORIGINS` | `*` | Allowed CORS origins |
 ### Nanonets-OCR (vLLM-based)
 | Variable | Default | Description |
 |----------|---------|-------------|
 | `MODEL_NAME` | `nanonets/Nanonets-OCR2-3B` | HuggingFace model ID |
 | `HOST` | `0.0.0.0` | API bind address |
 | `PORT` | `8000` | API port |
 | `MAX_MODEL_LEN` | `30000` | Maximum sequence length |
 | `GPU_MEMORY_UTILIZATION` | `0.9` | GPU memory usage (0-1) |
 ---
 ## 🏗️ Architecture Notes
 ### Dual-VLM Consensus Strategy
 For production document extraction, consider using multiple models together:
 1. **Pass 1:** MiniCPM-V visual extraction (images → JSON)
 2. **Pass 2:** Nanonets-OCR semantic extraction (images → markdown → JSON)
 3. **Consensus:** If results match → Done (fast path)
 4. **Pass 3+:** Additional visual passes if needed
 This dual-VLM approach catches extraction errors that single models miss.
 ### Why Multi-Model Works
 - **Different architectures:** Independent models cross-validate each other
 - **Specialized strengths:** Nanonets-OCR2-3B excels at document structure; MiniCPM-V handles general vision
 - **Native processing:** All VLMs see original images—no intermediate structure loss
 ### Model Selection Guide
 | Task | Recommended Model |
 |------|-------------------|
 | General image understanding | MiniCPM-V 4.5 |
 | Document OCR with structure preservation | Nanonets-OCR2-3B |
 | Complex visual reasoning / code generation | Qwen3-VL-30B |
 | Multi-image analysis | MiniCPM-V 4.5 |
 | Visual agent tasks | Qwen3-VL-30B |
 | Large documents (30K+ tokens) | Nanonets-OCR2-3B |
 ---
 ## 🔧 Building from Source
 ```bash
 # Clone the repository
@@ -139,9 +318,66 @@ cd ht-docker-ai
 ./build-images.sh
 # Run tests
-./test-images.sh
+pnpm test
 ```
-## License
+---
-MIT - Task Venture Capital GmbH
+## 🔍 Troubleshooting
 ### Model download hangs
 ```bash
 docker logs -f <container-name>
 ```
 Model downloads can take several minutes (~5GB for MiniCPM-V, ~20GB for Qwen3-VL).
 ### Out of memory
 - **GPU:** Use a lighter model variant or upgrade VRAM
 - **CPU:** Increase container memory: `--memory=16g`
 ### API not responding
 1. Check container health: `docker ps`
 2. Review logs: `docker logs <container>`
 3. Verify port: `curl localhost:11434/api/tags` or `curl localhost:8000/health`
 ### Enable NVIDIA GPU support on host
 ```bash
 # Install NVIDIA Container Toolkit
 curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
 curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
 sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
 sudo nvidia-ctk runtime configure --runtime=docker
 sudo systemctl restart docker
 ```
 ### GPU Memory Contention (Multi-Model)
 When running multiple VLMs on a single GPU:
 - vLLM and Ollama both need significant GPU memory
 - **Single GPU:** Run services sequentially (stop one before starting another)
 - **Multi-GPU:** Assign each service to a different GPU via `CUDA_VISIBLE_DEVICES`
 ---
 ## License and Legal Information
 This repository contains open-source code licensed under the MIT License. A copy of the license can be found in the [LICENSE](./LICENSE) file.
 **Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
 ### Trademarks
 This project is owned and maintained by Task Venture Capital GmbH. The names and logos associated with Task Venture Capital GmbH and any related products or services are trademarks of Task Venture Capital GmbH or third parties, and are not included within the scope of the MIT license granted herein.
 Use of these trademarks must comply with Task Venture Capital GmbH's Trademark Guidelines or the guidelines of the respective third-party owners, and any usage must be approved in writing. Third-party trademarks used herein are the property of their respective owners and used only in a descriptive manner, e.g. for an implementation of an API or similar.
 ### Company Information
 Task Venture Capital GmbH
 Registered at District Court Bremen HRB 35230 HB, Germany
 For any legal inquiries or further information, please contact us via email at hello@task.vc.
 By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
--- a/test/helpers/docker.ts
+++ b/test/helpers/docker.ts
@@ -0,0 +1,351 @@
 import { execSync } from 'child_process';
 // Project container names (only manage these)
 const PROJECT_CONTAINERS = [
  'minicpm-test',
  'nanonets-test',
 ];
 // Image configurations
 export interface IImageConfig {
  name: string;
  dockerfile: string;
  buildContext: string;
  containerName: string;
  ports: string[];
  volumes?: string[];
  gpus?: boolean;
  healthEndpoint?: string;
  healthTimeout?: number;
 }
 export const IMAGES = {
  minicpm: {
    name: 'minicpm45v',
    dockerfile: 'Dockerfile_minicpm45v_gpu',
    buildContext: '.',
    containerName: 'minicpm-test',
    ports: ['11434:11434'],
    volumes: ['ht-ollama-models:/root/.ollama'],
    gpus: true,
    healthEndpoint: 'http://localhost:11434/api/tags',
    healthTimeout: 120000,
  } as IImageConfig,
  // Nanonets-OCR2-3B - Document OCR optimized VLM (Qwen2.5-VL-3B fine-tuned, Oct 2025)
  nanonetsOcr: {
    name: 'nanonets-ocr',
    dockerfile: 'Dockerfile_nanonets_vllm_gpu_VRAM10GB',
    buildContext: '.',
    containerName: 'nanonets-test',
    ports: ['8000:8000'],
    volumes: ['ht-huggingface-cache:/root/.cache/huggingface'],
    gpus: true,
    healthEndpoint: 'http://localhost:8000/health',
    healthTimeout: 300000, // 5 minutes for model loading
  } as IImageConfig,
 };
 /**
 * Execute a shell command and return output
 */
 function exec(command: string, silent = false): string {
  try {
    return execSync(command, {
      encoding: 'utf-8',
      stdio: silent ? 'pipe' : 'inherit',
    });
  } catch (err: unknown) {
    if (silent) return '';
    throw err;
  }
 }
 /**
 * Check if a Docker image exists locally
 */
 export function imageExists(imageName: string): boolean {
  const result = exec(`docker images -q ${imageName}`, true);
  return result.trim().length > 0;
 }
 /**
 * Check if a container is running
 */
 export function isContainerRunning(containerName: string): boolean {
  const result = exec(`docker ps --filter "name=^${containerName}$" --format "{{.Names}}"`, true);
  return result.trim() === containerName;
 }
 /**
 * Check if a container exists (running or stopped)
 */
 export function containerExists(containerName: string): boolean {
  const result = exec(`docker ps -a --filter "name=^${containerName}$" --format "{{.Names}}"`, true);
  return result.trim() === containerName;
 }
 /**
 * Stop and remove a container
 */
 export function removeContainer(containerName: string): void {
  if (containerExists(containerName)) {
    console.log(`[Docker] Removing container: ${containerName}`);
    exec(`docker rm -f ${containerName}`, true);
  }
 }
 /**
 * Stop all project containers that conflict with the required one (port-based)
 */
 export function stopConflictingContainers(requiredContainer: string, requiredPort: string): void {
  // Stop project containers using the same port
  for (const container of PROJECT_CONTAINERS) {
    if (container === requiredContainer) continue;
    if (isContainerRunning(container)) {
      // Check if this container uses the same port
      const ports = exec(`docker port ${container} 2>/dev/null || true`, true);
      if (ports.includes(requiredPort.split(':')[0])) {
        console.log(`[Docker] Stopping conflicting container: ${container}`);
        exec(`docker stop ${container}`, true);
      }
    }
  }
 }
 /**
 * Stop all GPU-consuming project containers (for GPU memory management)
 * This ensures GPU memory is freed before starting a new GPU service
 */
 export function stopAllGpuContainers(exceptContainer?: string): void {
  for (const container of PROJECT_CONTAINERS) {
    if (container === exceptContainer) continue;
    if (isContainerRunning(container)) {
      console.log(`[Docker] Stopping GPU container: ${container}`);
      exec(`docker stop ${container}`, true);
      // Give the GPU a moment to free memory
    }
  }
  // Brief pause to allow GPU memory to be released
  execSync('sleep 2');
 }
 /**
 * Build a Docker image
 */
 export function buildImage(config: IImageConfig): void {
  console.log(`[Docker] Building image: ${config.name}`);
  const cmd = `docker build --load -f ${config.dockerfile} -t ${config.name} ${config.buildContext}`;
  exec(cmd);
 }
 /**
 * Start a container from an image
 */
 export function startContainer(config: IImageConfig): void {
  // Remove existing container if it exists
  removeContainer(config.containerName);
  console.log(`[Docker] Starting container: ${config.containerName}`);
  const portArgs = config.ports.map((p) => `-p ${p}`).join(' ');
  const volumeArgs = config.volumes?.map((v) => `-v ${v}`).join(' ') || '';
  const gpuArgs = config.gpus ? '--gpus all' : '';
  const cmd = `docker run -d --name ${config.containerName} ${gpuArgs} ${portArgs} ${volumeArgs} ${config.name}`;
  exec(cmd);
 }
 /**
 * Wait for a container to become healthy
 */
 export async function waitForHealth(
  endpoint: string,
  timeoutMs: number = 120000,
  intervalMs: number = 5000
 ): Promise<boolean> {
  const startTime = Date.now();
  console.log(`[Docker] Waiting for health: ${endpoint}`);
  while (Date.now() - startTime < timeoutMs) {
    try {
      const response = await fetch(endpoint, {
        method: 'GET',
        signal: AbortSignal.timeout(5000),
      });
      if (response.ok) {
        console.log(`[Docker] Service healthy!`);
        return true;
      }
    } catch {
      // Service not ready yet
    }
    const elapsed = Math.round((Date.now() - startTime) / 1000);
    console.log(`[Docker] Waiting... (${elapsed}s)`);
    await new Promise((resolve) => setTimeout(resolve, intervalMs));
  }
  console.log(`[Docker] Health check timeout after ${timeoutMs / 1000}s`);
  return false;
 }
 /**
 * Ensure a service is running and healthy
 * - Builds image if missing
 * - Stops conflicting project containers
 * - Starts container if not running
 * - Waits for health check
 */
 export async function ensureService(config: IImageConfig): Promise<boolean> {
  console.log(`\n[Docker] Ensuring service: ${config.name}`);
  // Build image if it doesn't exist
  if (!imageExists(config.name)) {
    console.log(`[Docker] Image not found, building...`);
    buildImage(config);
  }
  // For GPU services, stop ALL other GPU containers to free GPU memory
  if (config.gpus) {
    stopAllGpuContainers(config.containerName);
  }
  // Stop conflicting containers on the same port
  const mainPort = config.ports[0];
  stopConflictingContainers(config.containerName, mainPort);
  // Start container if not running
  if (!isContainerRunning(config.containerName)) {
    startContainer(config);
  } else {
    console.log(`[Docker] Container already running: ${config.containerName}`);
  }
  // Wait for health
  if (config.healthEndpoint) {
    return waitForHealth(config.healthEndpoint, config.healthTimeout);
  }
  return true;
 }
 /**
 * Ensure MiniCPM service is running (Ollama with GPU)
 */
 export async function ensureMiniCpm(): Promise<boolean> {
  return ensureService(IMAGES.minicpm);
 }
 /**
 * Check if GPU is available
 */
 export function isGpuAvailable(): boolean {
  try {
    const result = exec('nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null', true);
    return result.trim().length > 0;
  } catch {
    return false;
  }
 }
 /**
 * Ensure an Ollama model is pulled and available
 * Uses the MiniCPM container (which runs Ollama) to pull the model
 */
 export async function ensureOllamaModel(modelName: string): Promise<boolean> {
  const OLLAMA_URL = 'http://localhost:11434';
  console.log(`\n[Ollama] Ensuring model: ${modelName}`);
  // Check if model exists
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      // Exact match required - don't match on prefix
      const exists = models.some((m: { name: string }) => m.name === modelName);
      if (exists) {
        console.log(`[Ollama] Model already available: ${modelName}`);
        return true;
      }
    }
  } catch {
    console.log(`[Ollama] Cannot check models, Ollama may not be running`);
    return false;
  }
  // Pull the model
  console.log(`[Ollama] Pulling model: ${modelName} (this may take a while)...`);
  try {
    const response = await fetch(`${OLLAMA_URL}/api/pull`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ name: modelName, stream: false }),
    });
    if (response.ok) {
      console.log(`[Ollama] Model pulled successfully: ${modelName}`);
      return true;
    } else {
      console.log(`[Ollama] Failed to pull model: ${response.status}`);
      return false;
    }
  } catch (err) {
    console.log(`[Ollama] Error pulling model: ${err}`);
    return false;
  }
 }
 /**
 * Ensure Qwen2.5 7B model is available (for text-only JSON extraction)
 */
 export async function ensureQwen25(): Promise<boolean> {
  // First ensure the Ollama service (MiniCPM container) is running
  const ollamaOk = await ensureMiniCpm();
  if (!ollamaOk) return false;
  // Then ensure the Qwen2.5 model is pulled
  return ensureOllamaModel('qwen2.5:7b');
 }
 /**
 * Ensure Ministral 3 8B model is available (for structured JSON extraction)
 * Ministral 3 has native JSON output support and OCR-style document extraction
 */
 export async function ensureMinistral3(): Promise<boolean> {
  // First ensure the Ollama service (MiniCPM container) is running
  const ollamaOk = await ensureMiniCpm();
  if (!ollamaOk) return false;
  // Then ensure the Ministral 3 8B model is pulled
  return ensureOllamaModel('ministral-3:8b');
 }
 /**
 * Ensure Qwen3-VL 8B model is available (vision-language model)
 * Q4_K_M quantization (~5GB) - fits in 15GB VRAM with room to spare
 */
 export async function ensureQwen3Vl(): Promise<boolean> {
  // First ensure the Ollama service is running
  const ollamaOk = await ensureMiniCpm();
  if (!ollamaOk) return false;
  // Then ensure Qwen3-VL 8B is pulled
  return ensureOllamaModel('qwen3-vl:8b');
 }
 /**
 * Ensure Nanonets-OCR2-3B service is running (via vLLM)
 * Document OCR optimized VLM based on Qwen2.5-VL-3B (Oct 2025 release)
 */
 export async function ensureNanonetsOcr(): Promise<boolean> {
  if (!isGpuAvailable()) {
    console.log('[Docker] WARNING: Nanonets-OCR2-3B requires GPU, but none detected');
  }
  return ensureService(IMAGES.nanonetsOcr);
 }
--- a/test/test.bankstatements.minicpm.ts
+++ b/test/test.bankstatements.minicpm.ts
@@ -0,0 +1,436 @@
 /**
 * Bank statement extraction using MiniCPM-V via smartagent DualAgentOrchestrator
 *
 * Uses vision-capable orchestrator with JsonValidatorTool for self-validation:
 * 1. Process each page with the orchestrator
 * 2. Driver extracts transactions and validates JSON before completing
 * 3. Streaming output during extraction
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureMiniCpm } from './helpers/docker.js';
 import { SmartAi } from '@push.rocks/smartai';
 import { DualAgentOrchestrator, JsonValidatorTool } from '@push.rocks/smartagent';
 const OLLAMA_URL = 'http://localhost:11434';
 const MODEL = 'openbmb/minicpm-v4.5:q8_0';
 interface ITransaction {
  date: string;
  counterparty: string;
  amount: number;
 }
 // SmartAi instance and orchestrator (initialized in setup)
 let smartAi: SmartAi;
 let orchestrator: DualAgentOrchestrator;
 /**
 * Convert PDF to PNG images using ImageMagick
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    execSync(
      `convert -density 300 -quality 100 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f: string) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 const EXTRACTION_PROMPT = `Extract ALL transactions from this bank statement page as a JSON array.
 IMPORTANT RULES:
 1. Each transaction has: date, counterparty (description), and an amount
 2. Amount is NEGATIVE for money going OUT (debits, payments, withdrawals)
 3. Amount is POSITIVE for money coming IN (credits, deposits, refunds)
 4. Date format: YYYY-MM-DD
 5. Do NOT include: opening balance, closing balance, subtotals, headers, or summary rows
 6. Only include actual transactions with a specific date and amount
 Before completing, validate your JSON output:
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON_ARRAY_HERE"}</params>
 </tool_call>
 Output format (must be a valid JSON array):
 [
  {"date": "2021-06-01", "counterparty": "COMPANY NAME", "amount": -25.99},
  {"date": "2021-06-02", "counterparty": "DEPOSIT FROM", "amount": 100.00}
 ]
 Only complete after validation passes. Output the final JSON array in <task_complete> tags.`;
 /**
 * Parse amount from various formats
 */
 function parseAmount(value: unknown): number {
  if (typeof value === 'number') return value;
  if (typeof value !== 'string') return 0;
  let s = value.replace(/[€$£\s]/g, '').replace('−', '-').replace('–', '-');
  // European format: comma is decimal
  if (s.includes(',') && s.indexOf(',') > s.lastIndexOf('.')) {
    s = s.replace(/\./g, '').replace(',', '.');
  } else {
    s = s.replace(/,/g, '');
  }
  return parseFloat(s) || 0;
 }
 /**
 * Extract JSON from response (handles markdown code blocks and task_complete tags)
 */
 function extractJsonFromResponse(response: string): unknown[] | null {
  // Try to find JSON in task_complete tags
  const completeMatch = response.match(/<task_complete>([\s\S]*?)<\/task_complete>/);
  if (completeMatch) {
    const content = completeMatch[1].trim();
    // Try to find JSON in the content
    const codeBlockMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
    const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : content;
    try {
      const parsed = JSON.parse(jsonStr);
      if (Array.isArray(parsed)) return parsed;
    } catch {
      // Try to find JSON array pattern
      const jsonMatch = jsonStr.match(/\[[\s\S]*\]/);
      if (jsonMatch) {
        try {
          const parsed = JSON.parse(jsonMatch[0]);
          if (Array.isArray(parsed)) return parsed;
        } catch {
          return null;
        }
      }
    }
  }
  // Try to find JSON in markdown code block
  const codeBlockMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : response.trim();
  try {
    const parsed = JSON.parse(jsonStr);
    if (Array.isArray(parsed)) return parsed;
  } catch {
    // Try to find JSON array pattern
    const jsonMatch = jsonStr.match(/\[[\s\S]*\]/);
    if (jsonMatch) {
      try {
        const parsed = JSON.parse(jsonMatch[0]);
        if (Array.isArray(parsed)) return parsed;
      } catch {
        return null;
      }
    }
  }
  return null;
 }
 /**
 * Parse JSON response into transactions
 */
 function parseJsonToTransactions(response: string): ITransaction[] {
  const parsed = extractJsonFromResponse(response);
  if (!parsed || !Array.isArray(parsed)) return [];
  return parsed.map((tx: any) => ({
    date: String(tx.date || ''),
    counterparty: String(tx.counterparty || tx.description || ''),
    amount: parseAmount(tx.amount),
  }));
 }
 /**
 * Extract transactions from a single page using smartagent orchestrator
 */
 async function extractTransactionsFromPage(image: string, pageNum: number): Promise<ITransaction[]> {
  console.log(`\n    ======== Page ${pageNum} ========`);
  const startTime = Date.now();
  const result = await orchestrator.run(EXTRACTION_PROMPT, { images: [image] });
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  console.log(`\n    [Page ${pageNum}] Completed in ${elapsed}s (${result.iterations} iterations, status: ${result.status})`);
  const transactions = parseJsonToTransactions(result.result);
  console.log(`    [Page ${pageNum}] Extracted ${transactions.length} transactions:`);
  for (let i = 0; i < Math.min(transactions.length, 10); i++) {
    const tx = transactions[i];
    console.log(`      ${(i + 1).toString().padStart(2)}. ${tx.date} | ${tx.counterparty.substring(0, 30).padEnd(30)} | ${tx.amount >= 0 ? '+' : ''}${tx.amount.toFixed(2)}`);
  }
  if (transactions.length > 10) {
    console.log(`      ... and ${transactions.length - 10} more transactions`);
  }
  return transactions;
 }
 /**
 * Extract all transactions from bank statement
 */
 async function extractTransactions(images: string[]): Promise<ITransaction[]> {
  console.log(`    [Vision] Processing ${images.length} page(s) with smartagent DualAgentOrchestrator`);
  const allTransactions: ITransaction[] = [];
  for (let i = 0; i < images.length; i++) {
    const pageTransactions = await extractTransactionsFromPage(images[i], i + 1);
    allTransactions.push(...pageTransactions);
  }
  console.log(`    [Vision] Total: ${allTransactions.length} transactions`);
  return allTransactions;
 }
 /**
 * Compare extracted transactions against expected
 */
 function compareTransactions(
  extracted: ITransaction[],
  expected: ITransaction[]
 ): { matches: number; total: number; errors: string[]; variations: string[] } {
  const errors: string[] = [];
  const variations: string[] = [];
  let matches = 0;
  for (let i = 0; i < expected.length; i++) {
    const exp = expected[i];
    const ext = extracted[i];
    if (!ext) {
      errors.push(`Missing transaction ${i}: ${exp.date} ${exp.counterparty}`);
      continue;
    }
    const dateMatch = ext.date === exp.date;
    const amountMatch = Math.abs(ext.amount - exp.amount) < 0.01;
    if (dateMatch && amountMatch) {
      matches++;
      // Track counterparty variations (date and amount match but name differs)
      if (ext.counterparty !== exp.counterparty) {
        variations.push(
          `[${i}] "${exp.counterparty}" → "${ext.counterparty}"`
        );
      }
    } else {
      errors.push(
        `Mismatch at ${i}: expected ${exp.date}/${exp.amount}, got ${ext.date}/${ext.amount}`
      );
    }
  }
  if (extracted.length > expected.length) {
    errors.push(`Extra transactions: ${extracted.length - expected.length}`);
  }
  return { matches, total: expected.length, errors, variations };
 }
 /**
 * Find all test cases (PDF + JSON pairs) in .nogit/
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit');
  if (!fs.existsSync(testDir)) {
    return [];
  }
  const files = fs.readdirSync(testDir);
  const pdfFiles = files.filter((f: string) => f.endsWith('.pdf'));
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of pdfFiles) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases.sort((a, b) => a.name.localeCompare(b.name));
 }
 // Tests
 tap.test('setup: ensure Docker containers are running', async () => {
  console.log('\n[Setup] Checking Docker containers...\n');
  const minicpmOk = await ensureMiniCpm();
  expect(minicpmOk).toBeTrue();
  console.log('\n[Setup] All containers ready!\n');
 });
 tap.test('setup: initialize smartagent orchestrator', async () => {
  console.log('[Setup] Initializing SmartAi and DualAgentOrchestrator...');
  smartAi = new SmartAi({
    ollama: {
      baseUrl: OLLAMA_URL,
      model: MODEL,
      defaultOptions: {
        num_ctx: 32768,
        num_predict: 4000,
        temperature: 0.1,
      },
      defaultTimeout: 300000, // 5 minutes for vision tasks
    },
  });
  await smartAi.start();
  orchestrator = new DualAgentOrchestrator({
    smartAiInstance: smartAi,
    defaultProvider: 'ollama',
    guardianPolicyPrompt: `You are a Guardian agent overseeing bank statement extraction tasks.
 APPROVE all tool calls that:
 - Use the json.validate action to verify JSON output
 - Are reasonable attempts to complete the extraction task
 REJECT tool calls that:
 - Attempt to access external resources
 - Try to execute arbitrary code
 - Are clearly unrelated to bank statement extraction`,
    driverSystemMessage: `You are an AI assistant that extracts bank transactions from statement images.
 Your task is to analyze bank statement images and extract transaction data.
 You have access to a json.validate tool to verify your JSON output.
 IMPORTANT: Always validate your JSON before completing the task.
 ## Tool Usage Format
 When you need to validate JSON, output:
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON_ARRAY"}</params>
 </tool_call>
 ## Completion Format
 After validation passes, complete the task:
 <task_complete>
 [{"date": "YYYY-MM-DD", "counterparty": "...", "amount": -123.45}, ...]
 </task_complete>`,
    maxIterations: 5,
    maxConsecutiveRejections: 3,
    onToken: (token, source) => {
      if (source === 'driver') {
        process.stdout.write(token);
      }
    },
    onProgress: (event) => {
      if (event.logLevel === 'error') {
        console.error(event.logMessage);
      }
    },
  });
  // Register the JsonValidatorTool
  orchestrator.registerTool(new JsonValidatorTool());
  await orchestrator.start();
  console.log('[Setup] Orchestrator initialized!\n');
 });
 tap.test('should have MiniCPM-V model loaded', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  const data = await response.json();
  const modelNames = data.models.map((m: { name: string }) => m.name);
  expect(modelNames.some((name: string) => name.includes('minicpm'))).toBeTrue();
 });
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} bank statement test cases (smartagent + MiniCPM-V)\n`);
 let passedCount = 0;
 let failedCount = 0;
 for (const testCase of testCases) {
  tap.test(`should extract: ${testCase.name}`, async () => {
    const expected: ITransaction[] = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.length} transactions`);
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`  Pages: ${images.length}`);
    const extracted = await extractTransactions(images);
    console.log(`  Extracted: ${extracted.length} transactions`);
    const result = compareTransactions(extracted, expected);
    const perfectMatch = result.matches === result.total && extracted.length === expected.length;
    if (perfectMatch) {
      passedCount++;
      console.log(`  Result: PASS (${result.matches}/${result.total})`);
    } else {
      failedCount++;
      console.log(`  Result: FAIL (${result.matches}/${result.total})`);
      result.errors.slice(0, 10).forEach((e) => console.log(`    - ${e}`));
    }
    // Log counterparty variations (names that differ but date/amount matched)
    if (result.variations.length > 0) {
      console.log(`  Counterparty variations (${result.variations.length}):`);
      result.variations.slice(0, 5).forEach((v) => console.log(`    ${v}`));
      if (result.variations.length > 5) {
        console.log(`    ... and ${result.variations.length - 5} more variations`);
      }
    }
    expect(result.matches).toEqual(result.total);
    expect(extracted.length).toEqual(expected.length);
  });
 }
 tap.test('cleanup: stop orchestrator', async () => {
  if (orchestrator) {
    await orchestrator.stop();
  }
  console.log('[Cleanup] Orchestrator stopped');
 });
 tap.test('summary', async () => {
  const total = testCases.length;
  console.log(`\n======================================================`);
  console.log(`   Bank Statement Summary`);
  console.log(`   (smartagent + ${MODEL})`);
  console.log(`======================================================`);
  console.log(`  Method:   DualAgentOrchestrator with vision`);
  console.log(`  Passed:   ${passedCount}/${total}`);
  console.log(`  Failed:   ${failedCount}/${total}`);
  console.log(`======================================================\n`);
 });
 export default tap.start();
--- a/test/test.bankstatements.nanonets.ts
+++ b/test/test.bankstatements.nanonets.ts
@@ -0,0 +1,752 @@
 /**
 * Bank statement extraction using Nanonets-OCR2-3B + GPT-OSS 20B (sequential two-stage pipeline)
 *
 * Stage 1: Nanonets-OCR2-3B converts ALL document pages to markdown (stop after completion)
 * Stage 2: GPT-OSS 20B extracts structured JSON from saved markdown (after Nanonets stops)
 *
 * This approach avoids GPU contention by running services sequentially.
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureNanonetsOcr, ensureMiniCpm, isContainerRunning } from './helpers/docker.js';
 import { SmartAi } from '@push.rocks/smartai';
 import { DualAgentOrchestrator, JsonValidatorTool } from '@push.rocks/smartagent';
 const NANONETS_URL = 'http://localhost:8000/v1';
 const NANONETS_MODEL = 'nanonets/Nanonets-OCR2-3B';
 const OLLAMA_URL = 'http://localhost:11434';
 const EXTRACTION_MODEL = 'gpt-oss:20b';
 // Temp directory for storing markdown between stages
 const TEMP_MD_DIR = path.join(os.tmpdir(), 'nanonets-markdown');
 // SmartAi instance for Ollama with optimized settings
 const smartAi = new SmartAi({
  ollama: {
    baseUrl: OLLAMA_URL,
    model: EXTRACTION_MODEL,
    defaultOptions: {
      num_ctx: 32768,   // Larger context for long statements + thinking
      temperature: 0,   // Deterministic for JSON extraction
    },
    defaultTimeout: 600000,  // 10 minute timeout for large documents
  },
 });
 // DualAgentOrchestrator for structured task execution
 let orchestrator: DualAgentOrchestrator;
 interface ITransaction {
  date: string;
  counterparty: string;
  amount: number;
 }
 interface IImageData {
  base64: string;
  width: number;
  height: number;
  pageNum: number;
 }
 interface ITestCase {
  name: string;
  pdfPath: string;
  jsonPath: string;
  markdownPath?: string;
  images?: IImageData[];
 }
 // Nanonets-specific prompt for document OCR to markdown
 const NANONETS_OCR_PROMPT = `Extract the text from the above document as if you were reading it naturally.
 Return the tables in html format.
 Return the equations in LaTeX representation.
 If there is an image in the document and image caption is not present, add a small description inside <img></img> tag.
 Watermarks should be wrapped in brackets. Ex: <watermark>OFFICIAL COPY</watermark>.
 Page numbers should be wrapped in brackets. Ex: <page_number>14</page_number>.`;
 // JSON extraction prompt for GPT-OSS 20B (sent AFTER the statement text is provided)
 const JSON_EXTRACTION_PROMPT = `Extract ALL transactions from the bank statement. Return ONLY valid JSON array.
 WHERE TO FIND DATA:
 - Transactions are typically in TABLES with columns: Date, Description/Counterparty, Debit, Credit, Balance
 - Look for rows with actual money movements, NOT header rows or summary totals
 RULES:
 1. date: Convert to YYYY-MM-DD format
 2. counterparty: The name/description of who the money went to/from
 3. amount: NEGATIVE for debits/withdrawals, POSITIVE for credits/deposits
 4. Only include actual transactions, NOT opening/closing balances
 JSON array only:
 [{"date":"YYYY-MM-DD","counterparty":"NAME","amount":-25.99}]`;
 // Constants for smart batching
 const MAX_VISUAL_TOKENS = 28000;  // ~32K context minus prompt/output headroom
 const PATCH_SIZE = 14;  // Qwen2.5-VL uses 14x14 patches
 /**
 * Estimate visual tokens for an image based on dimensions
 */
 function estimateVisualTokens(width: number, height: number): number {
  return Math.ceil((width * height) / (PATCH_SIZE * PATCH_SIZE));
 }
 /**
 * Process images one page at a time for reliability
 */
 function batchImages(images: IImageData[]): IImageData[][] {
  // One page per batch for reliable processing
  return images.map(img => [img]);
 }
 /**
 * Convert PDF to JPEG images using ImageMagick with dimension tracking
 */
 function convertPdfToImages(pdfPath: string): IImageData[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.jpg');
  try {
    execSync(
      `convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f: string) => f.endsWith('.jpg')).sort();
    const images: IImageData[] = [];
    for (let i = 0; i < files.length; i++) {
      const file = files[i];
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      // Get image dimensions using identify command
      const dimensions = execSync(`identify -format "%w %h" "${imagePath}"`, { encoding: 'utf-8' }).trim();
      const [width, height] = dimensions.split(' ').map(Number);
      images.push({
        base64: imageData.toString('base64'),
        width,
        height,
        pageNum: i + 1,
      });
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Convert a batch of pages to markdown using Nanonets-OCR-s
 */
 async function convertBatchToMarkdown(batch: IImageData[]): Promise<string> {
  const startTime = Date.now();
  const pageNums = batch.map(img => img.pageNum).join(', ');
  // Build content array with all images first, then the prompt
  const content: Array<{ type: string; image_url?: { url: string }; text?: string }> = [];
  for (const img of batch) {
    content.push({
      type: 'image_url',
      image_url: { url: `data:image/jpeg;base64,${img.base64}` },
    });
  }
  // Add prompt with page separator instruction if multiple pages
  const promptText = batch.length > 1
    ? `${NANONETS_OCR_PROMPT}\n\nPlease clearly separate each page's content with "--- PAGE N ---" markers, where N is the page number starting from ${batch[0].pageNum}.`
    : NANONETS_OCR_PROMPT;
  content.push({ type: 'text', text: promptText });
  const response = await fetch(`${NANONETS_URL}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': 'Bearer dummy',
    },
    body: JSON.stringify({
      model: NANONETS_MODEL,
      messages: [{
        role: 'user',
        content,
      }],
      max_tokens: 4096 * batch.length,  // Scale output tokens with batch size
      temperature: 0.0,
    }),
    signal: AbortSignal.timeout(600000),  // 10 minute timeout for OCR
  });
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(`Nanonets API error: ${response.status} - ${errorText}`);
  }
  const data = await response.json();
  let responseContent = (data.choices?.[0]?.message?.content || '').trim();
  // For single-page batches, add page marker if not present
  if (batch.length === 1 && !responseContent.includes('--- PAGE')) {
    responseContent = `--- PAGE ${batch[0].pageNum} ---\n${responseContent}`;
  }
  console.log(`        Pages [${pageNums}]: ${responseContent.length} chars (${elapsed}s)`);
  return responseContent;
 }
 /**
 * Convert all pages of a document to markdown using smart batching
 */
 async function convertDocumentToMarkdown(images: IImageData[], docName: string): Promise<string> {
  const batches = batchImages(images);
  console.log(`      [${docName}] Processing ${images.length} page(s) in ${batches.length} batch(es)...`);
  const markdownParts: string[] = [];
  for (let i = 0; i < batches.length; i++) {
    const batch = batches[i];
    const batchTokens = batch.reduce((sum, img) => sum + estimateVisualTokens(img.width, img.height), 0);
    console.log(`        Batch ${i + 1}: ${batch.length} page(s), ~${batchTokens} tokens`);
    const markdown = await convertBatchToMarkdown(batch);
    markdownParts.push(markdown);
  }
  const fullMarkdown = markdownParts.join('\n\n');
  console.log(`      [${docName}] Complete: ${fullMarkdown.length} chars total`);
  return fullMarkdown;
 }
 /**
 * Stop Nanonets container
 */
 function stopNanonets(): void {
  console.log('    [Docker] Stopping Nanonets container...');
  try {
    execSync('docker stop nanonets-test 2>/dev/null || true', { stdio: 'pipe' });
    // Wait for GPU memory to be released
    execSync('sleep 5', { stdio: 'pipe' });
    console.log('    [Docker] Nanonets stopped');
  } catch {
    console.log('    [Docker] Nanonets was not running');
  }
 }
 /**
 * Ensure GPT-OSS 20B model is available and warmed up
 */
 async function ensureExtractionModel(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === EXTRACTION_MODEL)) {
        console.log(`    [Ollama] Model available: ${EXTRACTION_MODEL}`);
        return true;
      }
    }
  } catch {
    return false;
  }
  console.log(`    [Ollama] Pulling ${EXTRACTION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: EXTRACTION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 /**
 * Try to extract valid JSON from a response string
 */
 function tryExtractJson(response: string): unknown[] | null {
  // Remove thinking tags
  let clean = response.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  // Try task_complete tags first
  const completeMatch = clean.match(/<task_complete>([\s\S]*?)<\/task_complete>/);
  if (completeMatch) {
    clean = completeMatch[1].trim();
  }
  // Try code block
  const codeBlockMatch = clean.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : clean;
  try {
    const parsed = JSON.parse(jsonStr);
    if (Array.isArray(parsed)) return parsed;
  } catch {
    // Try to find JSON array
    const jsonMatch = jsonStr.match(/\[[\s\S]*\]/);
    if (jsonMatch) {
      try {
        const parsed = JSON.parse(sanitizeJson(jsonMatch[0]));
        if (Array.isArray(parsed)) return parsed;
      } catch {
        return null;
      }
    }
    return null;
  }
  return null;
 }
 /**
 * Extract transactions from markdown using smartagent DualAgentOrchestrator
 * Validates JSON and retries if invalid
 */
 async function extractTransactionsFromMarkdown(markdown: string, queryId: string): Promise<ITransaction[]> {
  const startTime = Date.now();
  console.log(`      [${queryId}] Statement: ${markdown.length} chars`);
  // Build the extraction task with document context
  const taskPrompt = `Extract all transactions from this bank statement document and output ONLY the JSON array:
 ${markdown}
 ${JSON_EXTRACTION_PROMPT}
 Before completing, validate your JSON using the json.validate tool:
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON_ARRAY_HERE"}</params>
 </tool_call>
 Only complete after validation passes. Output the final JSON array in <task_complete></task_complete> tags.`;
  try {
    const result = await orchestrator.run(taskPrompt);
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] Status: ${result.status}, Iterations: ${result.iterations} (${elapsed}s)`);
    // Try to parse JSON from result
    let jsonData: unknown[] | null = null;
    let responseText = result.result || '';
    if (result.success && responseText) {
      jsonData = tryExtractJson(responseText);
    }
    // Fallback: try parsing from history
    if (!jsonData && result.history?.length > 0) {
      const lastMessage = result.history[result.history.length - 1];
      if (lastMessage?.content) {
        responseText = lastMessage.content;
        jsonData = tryExtractJson(responseText);
      }
    }
    if (!jsonData) {
      console.log(`      [${queryId}] Failed to parse JSON`);
      return [];
    }
    // Convert to transactions
    const txs = jsonData.map((tx: any) => ({
      date: String(tx.date || ''),
      counterparty: String(tx.counterparty || tx.description || ''),
      amount: parseAmount(tx.amount),
    }));
    console.log(`      [${queryId}] Parsed ${txs.length} transactions`);
    return txs;
  } catch (error) {
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] ERROR: ${error} (${elapsed}s)`);
    throw error;
  }
 }
 /**
 * Sanitize JSON string
 */
 function sanitizeJson(jsonStr: string): string {
  let s = jsonStr;
  s = s.replace(/"amount"\s*:\s*\+/g, '"amount": ');
  s = s.replace(/:\s*\+(\d)/g, ': $1');
  s = s.replace(/"amount"\s*:\s*(-?)(\d{1,3})\.(\d{3})\.(\d{2})\b/g, '"amount": $1$2$3.$4');
  s = s.replace(/,\s*([}\]])/g, '$1');
  s = s.replace(/"([^"\\]*)\n([^"]*)"/g, '"$1 $2"');
  s = s.replace(/"([^"\\]*)\t([^"]*)"/g, '"$1 $2"');
  s = s.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, ' ');
  return s;
 }
 /**
 * Parse amount from various formats
 */
 function parseAmount(value: unknown): number {
  if (typeof value === 'number') return value;
  if (typeof value !== 'string') return 0;
  let s = value.replace(/[€$£\s]/g, '').replace('−', '-').replace('–', '-');
  if (s.includes(',') && s.indexOf(',') > s.lastIndexOf('.')) {
    s = s.replace(/\./g, '').replace(',', '.');
  } else {
    s = s.replace(/,/g, '');
  }
  return parseFloat(s) || 0;
 }
 /**
 * Parse JSON response into transactions
 */
 function parseJsonResponse(response: string, queryId: string): ITransaction[] {
  // Remove thinking tags if present
  let cleanResponse = response.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  // Debug: show what we're working with
  console.log(`      [${queryId}] Response preview: ${cleanResponse.substring(0, 300)}...`);
  const codeBlockMatch = cleanResponse.match(/```(?:json)?\s*([\s\S]*?)```/);
  let jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : cleanResponse;
  jsonStr = sanitizeJson(jsonStr);
  try {
    const parsed = JSON.parse(jsonStr);
    if (Array.isArray(parsed)) {
      const txs = parsed.map(tx => ({
        date: String(tx.date || ''),
        counterparty: String(tx.counterparty || tx.description || ''),
        amount: parseAmount(tx.amount),
      }));
      console.log(`      [${queryId}] Parsed ${txs.length} transactions`);
      return txs;
    }
  } catch (e) {
    // Try to find a JSON array in the text
    const arrayMatch = jsonStr.match(/\[[\s\S]*\]/);
    if (arrayMatch) {
      console.log(`      [${queryId}] Array match found: ${arrayMatch[0].length} chars`);
      try {
        const parsed = JSON.parse(sanitizeJson(arrayMatch[0]));
        if (Array.isArray(parsed)) {
          const txs = parsed.map(tx => ({
            date: String(tx.date || ''),
            counterparty: String(tx.counterparty || tx.description || ''),
            amount: parseAmount(tx.amount),
          }));
          console.log(`      [${queryId}] Parsed ${txs.length} transactions (array match)`);
          return txs;
        }
      } catch (innerErr) {
        console.log(`      [${queryId}] Array parse error: ${(innerErr as Error).message}`);
      }
    } else {
      console.log(`      [${queryId}] No JSON array found in response`);
    }
  }
  console.log(`      [${queryId}] PARSE FAILED`);
  return [];
 }
 /**
 * Extract transactions (single pass)
 */
 async function extractTransactions(markdown: string, docName: string): Promise<ITransaction[]> {
  console.log(`    [${docName}] Extracting...`);
  const txs = await extractTransactionsFromMarkdown(markdown, docName);
  console.log(`    [${docName}] Extracted ${txs.length} transactions`);
  return txs;
 }
 /**
 * Compare transactions
 */
 function compareTransactions(
  extracted: ITransaction[],
  expected: ITransaction[]
 ): { matches: number; total: number; errors: string[] } {
  const errors: string[] = [];
  let matches = 0;
  for (let i = 0; i < expected.length; i++) {
    const exp = expected[i];
    const ext = extracted[i];
    if (!ext) {
      errors.push(`Missing tx ${i}: ${exp.date} ${exp.counterparty}`);
      continue;
    }
    const dateMatch = ext.date === exp.date;
    const amountMatch = Math.abs(ext.amount - exp.amount) < 0.01;
    if (dateMatch && amountMatch) {
      matches++;
    } else {
      errors.push(`Mismatch ${i}: exp ${exp.date}/${exp.amount}, got ${ext.date}/${ext.amount}`);
    }
  }
  if (extracted.length > expected.length) {
    errors.push(`Extra transactions: ${extracted.length - expected.length}`);
  }
  return { matches, total: expected.length, errors };
 }
 /**
 * Find all test cases
 */
 function findTestCases(): ITestCase[] {
  const testDir = path.join(process.cwd(), '.nogit');
  if (!fs.existsSync(testDir)) return [];
  const files = fs.readdirSync(testDir);
  const testCases: ITestCase[] = [];
  for (const pdf of files.filter((f: string) => f.endsWith('.pdf'))) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases.sort((a, b) => a.name.localeCompare(b.name));
 }
 // ============ TESTS ============
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} bank statement test cases\n`);
 // Ensure temp directory exists
 if (!fs.existsSync(TEMP_MD_DIR)) {
  fs.mkdirSync(TEMP_MD_DIR, { recursive: true });
 }
 // -------- STAGE 1: OCR with Nanonets --------
 // Check if all markdown files already exist
 function allMarkdownFilesExist(): boolean {
  for (const tc of testCases) {
    const mdPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
    if (!fs.existsSync(mdPath)) {
      return false;
    }
  }
  return true;
 }
 // Track whether we need to run Stage 1
 let stage1Needed = !allMarkdownFilesExist();
 tap.test('Stage 1: Setup Nanonets', async () => {
  console.log('\n========== STAGE 1: Nanonets OCR ==========\n');
  if (!stage1Needed) {
    console.log('  [SKIP] All markdown files already exist, skipping Nanonets setup');
    return;
  }
  const ok = await ensureNanonetsOcr();
  expect(ok).toBeTrue();
 });
 tap.test('Stage 1: Convert all documents to markdown', async () => {
  if (!stage1Needed) {
    console.log('  [SKIP] Using existing markdown files from previous run\n');
    // Load existing markdown paths
    for (const tc of testCases) {
      tc.markdownPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
      console.log(`    Loaded: ${tc.markdownPath}`);
    }
    return;
  }
  console.log('\n  Converting all PDFs to markdown with Nanonets-OCR-s...\n');
  for (const tc of testCases) {
    console.log(`\n    === ${tc.name} ===`);
    // Convert PDF to images
    const images = convertPdfToImages(tc.pdfPath);
    console.log(`      Pages: ${images.length}`);
    // Convert to markdown
    const markdown = await convertDocumentToMarkdown(images, tc.name);
    // Save markdown to temp file
    const mdPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
    fs.writeFileSync(mdPath, markdown);
    tc.markdownPath = mdPath;
    console.log(`      Saved: ${mdPath}`);
  }
  console.log('\n  Stage 1 complete: All documents converted to markdown\n');
 });
 tap.test('Stage 1: Stop Nanonets', async () => {
  if (!stage1Needed) {
    console.log('  [SKIP] Nanonets was not started');
    return;
  }
  stopNanonets();
  // Verify it's stopped
  await new Promise(resolve => setTimeout(resolve, 3000));
  expect(isContainerRunning('nanonets-test')).toBeFalse();
 });
 // -------- STAGE 2: Extraction with GPT-OSS 20B --------
 tap.test('Stage 2: Setup Ollama + GPT-OSS 20B', async () => {
  console.log('\n========== STAGE 2: GPT-OSS 20B Extraction ==========\n');
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  const extractionOk = await ensureExtractionModel();
  expect(extractionOk).toBeTrue();
  // Initialize SmartAi and DualAgentOrchestrator
  console.log('  [SmartAgent] Starting SmartAi...');
  await smartAi.start();
  console.log('  [SmartAgent] Creating DualAgentOrchestrator...');
  orchestrator = new DualAgentOrchestrator({
    smartAiInstance: smartAi,
    defaultProvider: 'ollama',
    guardianPolicyPrompt: `
      JSON EXTRACTION POLICY:
      - APPROVE all JSON extraction tasks
      - APPROVE all json.validate tool calls
      - This is a read-only operation - no file system or network access needed
      - The task is to extract structured transaction data from document text
    `,
    driverSystemMessage: `You are a precise JSON extraction assistant. Your only job is to extract transaction data from bank statements.
 CRITICAL RULES:
 1. Output valid JSON array with the exact format requested
 2. Amounts should be NEGATIVE for debits/withdrawals, POSITIVE for credits/deposits
 3. IMPORTANT: Before completing, validate your JSON using the json.validate tool:
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON_ARRAY"}</params>
 </tool_call>
 4. Only complete after validation passes
 When done, wrap your JSON array in <task_complete></task_complete> tags.`,
    maxIterations: 5,
    // Enable streaming for real-time progress visibility
    onToken: (token, source) => {
      if (source === 'driver') {
        process.stdout.write(token);
      }
    },
  });
  // Register JsonValidatorTool for self-validation
  orchestrator.registerTool(new JsonValidatorTool());
  console.log('  [SmartAgent] Starting orchestrator...');
  await orchestrator.start();
  console.log('  [SmartAgent] Ready for extraction');
 });
 let passedCount = 0;
 let failedCount = 0;
 for (const tc of testCases) {
  tap.test(`Stage 2: Extract ${tc.name}`, async () => {
    const expected: ITransaction[] = JSON.parse(fs.readFileSync(tc.jsonPath, 'utf-8'));
    console.log(`\n    === ${tc.name} ===`);
    console.log(`    Expected: ${expected.length} transactions`);
    // Load saved markdown
    const mdPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
    if (!fs.existsSync(mdPath)) {
      throw new Error(`Markdown not found: ${mdPath}. Run Stage 1 first.`);
    }
    const markdown = fs.readFileSync(mdPath, 'utf-8');
    console.log(`    Markdown: ${markdown.length} chars`);
    // Extract transactions (single pass)
    const extracted = await extractTransactions(markdown, tc.name);
    // Log results
    console.log(`    Extracted: ${extracted.length} transactions`);
    for (let i = 0; i < Math.min(extracted.length, 5); i++) {
      const tx = extracted[i];
      console.log(`      ${i + 1}. ${tx.date} | ${tx.counterparty.substring(0, 25).padEnd(25)} | ${tx.amount >= 0 ? '+' : ''}${tx.amount.toFixed(2)}`);
    }
    if (extracted.length > 5) {
      console.log(`      ... and ${extracted.length - 5} more`);
    }
    // Compare
    const result = compareTransactions(extracted, expected);
    const pass = result.matches === result.total && extracted.length === expected.length;
    if (pass) {
      passedCount++;
      console.log(`    Result: PASS (${result.matches}/${result.total})`);
    } else {
      failedCount++;
      console.log(`    Result: FAIL (${result.matches}/${result.total})`);
      result.errors.slice(0, 5).forEach(e => console.log(`      - ${e}`));
    }
    expect(result.matches).toEqual(result.total);
    expect(extracted.length).toEqual(expected.length);
  });
 }
 tap.test('Summary', async () => {
  // Cleanup orchestrator and SmartAi
  if (orchestrator) {
    console.log('\n  [SmartAgent] Stopping orchestrator...');
    await orchestrator.stop();
  }
  console.log('  [SmartAgent] Stopping SmartAi...');
  await smartAi.stop();
  console.log(`\n======================================================`);
  console.log(`   Bank Statement Summary (Nanonets + SmartAgent)`);
  console.log(`======================================================`);
  console.log(`  Stage 1:  Nanonets-OCR-s (document -> markdown)`);
  console.log(`  Stage 2:  GPT-OSS 20B + SmartAgent (markdown -> JSON)`);
  console.log(`  Passed:   ${passedCount}/${testCases.length}`);
  console.log(`  Failed:   ${failedCount}/${testCases.length}`);
  console.log(`======================================================\n`);
  // Only cleanup temp files if ALL tests passed
  if (failedCount === 0 && passedCount === testCases.length) {
    try {
      fs.rmSync(TEMP_MD_DIR, { recursive: true, force: true });
      console.log(`  Cleaned up temp directory: ${TEMP_MD_DIR}\n`);
    } catch {
      // Ignore
    }
  } else {
    console.log(`  Keeping temp directory for debugging: ${TEMP_MD_DIR}\n`);
  }
 });
 export default tap.start();
--- a/test/test.bankstatements.qwen3vl.ts
+++ b/test/test.bankstatements.qwen3vl.ts
@@ -0,0 +1,345 @@
 /**
 * Bank statement extraction using Qwen3-VL 8B Vision (Direct)
 *
 * Multi-query approach:
 * 1. First ask how many transactions on each page
 * 2. Then query each transaction individually
 * Single pass, no consensus voting.
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureMiniCpm } from './helpers/docker.js';
 const OLLAMA_URL = 'http://localhost:11434';
 const VISION_MODEL = 'qwen3-vl:8b';
 interface ITransaction {
  date: string;
  counterparty: string;
  amount: number;
 }
 /**
 * Convert PDF to PNG images
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    execSync(
      `convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f: string) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Query Qwen3-VL with a simple prompt
 */
 async function queryVision(image: string, prompt: string): Promise<string> {
  const response = await fetch(`${OLLAMA_URL}/api/chat`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: VISION_MODEL,
      messages: [{
        role: 'user',
        content: prompt,
        images: [image],
      }],
      stream: false,
      options: {
        num_predict: 500,
        temperature: 0.1,
      },
    }),
  });
  if (!response.ok) {
    throw new Error(`Ollama API error: ${response.status}`);
  }
  const data = await response.json();
  return (data.message?.content || '').trim();
 }
 /**
 * Count transactions on a page
 */
 async function countTransactions(image: string, pageNum: number): Promise<number> {
  const response = await queryVision(image,
    `How many transaction rows are in this bank statement table?
 Count only the data rows (with dates like "01.01.2024" and amounts like "- 50,00 €").
 Do NOT count the header row or summary/total rows.
 Answer with just the number, for example: 7`
  );
  console.log(`    [Page ${pageNum}] Count query response: "${response}"`);
  const match = response.match(/(\d+)/);
  const count = match ? parseInt(match[1], 10) : 0;
  console.log(`    [Page ${pageNum}] Parsed count: ${count}`);
  return count;
 }
 /**
 * Get a single transaction by index (logs immediately when complete)
 */
 async function getTransaction(image: string, index: number, pageNum: number): Promise<ITransaction | null> {
  const response = await queryVision(image,
    `This is a bank statement. Look at transaction row #${index} in the table (counting from top, excluding headers).
 Extract this transaction's details:
 - Date in YYYY-MM-DD format
 - Counterparty/description name
 - Amount as number (negative for debits like "- 21,47 €" = -21.47, positive for credits like "+ 100,00 €" = 100.00)
 Answer in format: DATE|COUNTERPARTY|AMOUNT
 Example: 2024-01-15|Amazon|−25.99`
  );
  // Parse the response
  const lines = response.split('\n').filter(l => l.includes('|'));
  const line = lines[lines.length - 1] || response;
  const parts = line.split('|').map(p => p.trim());
  if (parts.length >= 3) {
    // Parse amount - handle various formats
    let amountStr = parts[2].replace(/[€$£\s]/g, '').replace('−', '-').replace('–', '-');
    // European format: comma is decimal
    if (amountStr.includes(',')) {
      amountStr = amountStr.replace(/\./g, '').replace(',', '.');
    }
    const amount = parseFloat(amountStr) || 0;
    const tx = {
      date: parts[0],
      counterparty: parts[1],
      amount: amount,
    };
    // Log immediately as this transaction completes
    console.log(`    [P${pageNum} Tx${index.toString().padStart(2, ' ')}] ${tx.date} | ${tx.counterparty.substring(0, 25).padEnd(25)} | ${tx.amount >= 0 ? '+' : ''}${tx.amount.toFixed(2)}`);
    return tx;
  }
  // Log raw response on parse failure
  console.log(`    [P${pageNum} Tx${index.toString().padStart(2, ' ')}] PARSE FAILED: "${response.replace(/\n/g, ' ').substring(0, 60)}..."`);
  return null;
 }
 /**
 * Extract transactions from a single page using multi-query approach
 */
 async function extractTransactionsFromPage(image: string, pageNum: number): Promise<ITransaction[]> {
  // Step 1: Count transactions
  const count = await countTransactions(image, pageNum);
  if (count === 0) {
    return [];
  }
  // Step 2: Query each transaction (in batches to avoid overwhelming)
  // Each transaction logs itself as it completes
  const transactions: ITransaction[] = [];
  const batchSize = 5;
  for (let start = 1; start <= count; start += batchSize) {
    const end = Math.min(start + batchSize - 1, count);
    const indices = Array.from({ length: end - start + 1 }, (_, i) => start + i);
    // Query batch in parallel - each logs as it completes
    const results = await Promise.all(
      indices.map(i => getTransaction(image, i, pageNum))
    );
    for (const tx of results) {
      if (tx) {
        transactions.push(tx);
      }
    }
  }
  console.log(`    [Page ${pageNum}] Complete: ${transactions.length}/${count} extracted`);
  return transactions;
 }
 /**
 * Extract all transactions from bank statement
 */
 async function extractTransactions(images: string[]): Promise<ITransaction[]> {
  console.log(`    [Vision] Processing ${images.length} page(s) with Qwen3-VL (multi-query)`);
  const allTransactions: ITransaction[] = [];
  for (let i = 0; i < images.length; i++) {
    const pageTransactions = await extractTransactionsFromPage(images[i], i + 1);
    allTransactions.push(...pageTransactions);
  }
  console.log(`    [Vision] Total: ${allTransactions.length} transactions`);
  return allTransactions;
 }
 /**
 * Compare transactions
 */
 function compareTransactions(
  extracted: ITransaction[],
  expected: ITransaction[]
 ): { matches: number; total: number; errors: string[] } {
  const errors: string[] = [];
  let matches = 0;
  for (let i = 0; i < expected.length; i++) {
    const exp = expected[i];
    const ext = extracted[i];
    if (!ext) {
      errors.push(`Missing transaction ${i}: ${exp.date} ${exp.counterparty}`);
      continue;
    }
    const dateMatch = ext.date === exp.date;
    const amountMatch = Math.abs(ext.amount - exp.amount) < 0.01;
    if (dateMatch && amountMatch) {
      matches++;
    } else {
      errors.push(`Mismatch at ${i}: expected ${exp.date}/${exp.amount}, got ${ext.date}/${ext.amount}`);
    }
  }
  if (extracted.length > expected.length) {
    errors.push(`Extra transactions: ${extracted.length - expected.length}`);
  }
  return { matches, total: expected.length, errors };
 }
 /**
 * Find test cases in .nogit/
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit');
  if (!fs.existsSync(testDir)) return [];
  const files = fs.readdirSync(testDir);
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of files.filter((f: string) => f.endsWith('.pdf'))) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases.sort((a, b) => a.name.localeCompare(b.name));
 }
 /**
 * Ensure Qwen3-VL model is available
 */
 async function ensureQwen3Vl(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === VISION_MODEL)) {
        console.log(`[Ollama] Model available: ${VISION_MODEL}`);
        return true;
      }
    }
  } catch {
    return false;
  }
  console.log(`[Ollama] Pulling ${VISION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: VISION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 // Tests
 tap.test('setup: ensure Qwen3-VL is running', async () => {
  console.log('\n[Setup] Checking Qwen3-VL 8B...\n');
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  const visionOk = await ensureQwen3Vl();
  expect(visionOk).toBeTrue();
  console.log('\n[Setup] Ready!\n');
 });
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} bank statement test cases (Qwen3-VL)\n`);
 let passedCount = 0;
 let failedCount = 0;
 for (const testCase of testCases) {
  tap.test(`should extract: ${testCase.name}`, async () => {
    const expected: ITransaction[] = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.length} transactions`);
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`  Pages: ${images.length}`);
    const extracted = await extractTransactions(images);
    console.log(`  Extracted: ${extracted.length} transactions`);
    const result = compareTransactions(extracted, expected);
    const accuracy = result.total > 0 ? result.matches / result.total : 0;
    if (accuracy >= 0.95 && extracted.length === expected.length) {
      passedCount++;
      console.log(`  Result: PASS (${result.matches}/${result.total})`);
    } else {
      failedCount++;
      console.log(`  Result: FAIL (${result.matches}/${result.total})`);
      result.errors.slice(0, 5).forEach((e) => console.log(`    - ${e}`));
    }
    expect(accuracy).toBeGreaterThan(0.95);
    expect(extracted.length).toEqual(expected.length);
  });
 }
 tap.test('summary', async () => {
  const total = testCases.length;
  console.log(`\n======================================================`);
  console.log(`   Bank Statement Summary (Qwen3-VL Vision)`);
  console.log(`======================================================`);
  console.log(`  Method:   Multi-query (count then extract each)`);
  console.log(`  Passed:   ${passedCount}/${total}`);
  console.log(`  Failed:   ${failedCount}/${total}`);
  console.log(`======================================================\n`);
 });
 export default tap.start();
--- a/test/test.invoices.extraction.ts
+++ b/test/test.invoices.extraction.ts
@@ -0,0 +1,440 @@
 /**
 * Invoice extraction tuning - uses pre-generated markdown files
 *
 * Skips OCR stage, only runs GPT-OSS extraction on existing .debug.md files.
 * Use this to quickly iterate on extraction prompts and logic.
 *
 * Run with: tstest test/test.invoices.extraction.ts --verbose
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { ensureMiniCpm } from './helpers/docker.js';
 const OLLAMA_URL = 'http://localhost:11434';
 const EXTRACTION_MODEL = 'gpt-oss:20b';
 // Test these specific invoices (must have .debug.md files)
 const TEST_INVOICES = [
  'consensus_2021-09',
  'hetzner_2022-04',
  'qonto_2021-08',
  'qonto_2021-09',
 ];
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 interface ITestCase {
  name: string;
  markdownPath: string;
  jsonPath: string;
 }
 // JSON extraction prompt for GPT-OSS 20B (sent AFTER the invoice text is provided)
 const JSON_EXTRACTION_PROMPT = `Extract key fields from the invoice. Return ONLY valid JSON.
 WHERE TO FIND DATA:
 - invoice_number, invoice_date, vendor_name: Look in the HEADER section at the TOP of PAGE 1 (near "Invoice no.", "Invoice date:", "Rechnungsnummer")
 - net_amount, vat_amount, total_amount: Look in the SUMMARY section at the BOTTOM (look for "Total", "Amount due", "Gesamtbetrag")
 RULES:
 1. invoice_number: Extract ONLY the value (e.g., "R0015632540"), NOT the label "Invoice no."
 2. invoice_date: Convert to YYYY-MM-DD format (e.g., "14/04/2022" → "2022-04-14")
 3. vendor_name: The company issuing the invoice
 4. currency: EUR, USD, or GBP
 5. net_amount: Total before tax
 6. vat_amount: Tax amount
 7. total_amount: Final total with tax
 JSON only:
 {"invoice_number":"X","invoice_date":"YYYY-MM-DD","vendor_name":"X","currency":"EUR","net_amount":0,"vat_amount":0,"total_amount":0}`;
 /**
 * Ensure GPT-OSS 20B model is available
 */
 async function ensureExtractionModel(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === EXTRACTION_MODEL)) {
        console.log(`    [Ollama] Model available: ${EXTRACTION_MODEL}`);
        return true;
      }
    }
  } catch {
    return false;
  }
  console.log(`    [Ollama] Pulling ${EXTRACTION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: EXTRACTION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 /**
 * Parse amount from string (handles European format)
 */
 function parseAmount(s: string | number | undefined): number {
  if (s === undefined || s === null) return 0;
  if (typeof s === 'number') return s;
  const match = s.match(/([\d.,]+)/);
  if (!match) return 0;
  const numStr = match[1];
  const normalized = numStr.includes(',') && numStr.indexOf(',') > numStr.lastIndexOf('.')
    ? numStr.replace(/\./g, '').replace(',', '.')
    : numStr.replace(/,/g, '');
  return parseFloat(normalized) || 0;
 }
 /**
 * Extract invoice number - minimal normalization
 */
 function extractInvoiceNumber(s: string | undefined): string {
  if (!s) return '';
  return s.replace(/\*\*/g, '').replace(/`/g, '').trim();
 }
 /**
 * Extract date (YYYY-MM-DD) from response
 */
 function extractDate(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const isoMatch = clean.match(/(\d{4}-\d{2}-\d{2})/);
  if (isoMatch) return isoMatch[1];
  const dmyMatch = clean.match(/(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})/);
  if (dmyMatch) {
    return `${dmyMatch[3]}-${dmyMatch[2].padStart(2, '0')}-${dmyMatch[1].padStart(2, '0')}`;
  }
  return clean.replace(/[^\d-]/g, '').trim();
 }
 /**
 * Extract currency
 */
 function extractCurrency(s: string | undefined): string {
  if (!s) return 'EUR';
  const upper = s.toUpperCase();
  if (upper.includes('EUR') || upper.includes('€')) return 'EUR';
  if (upper.includes('USD') || upper.includes('$')) return 'USD';
  if (upper.includes('GBP') || upper.includes('£')) return 'GBP';
  return 'EUR';
 }
 /**
 * Extract JSON from response
 */
 function extractJsonFromResponse(response: string): Record<string, unknown> | null {
  let cleanResponse = response.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  const codeBlockMatch = cleanResponse.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : cleanResponse;
  try {
    return JSON.parse(jsonStr);
  } catch {
    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      try {
        return JSON.parse(jsonMatch[0]);
      } catch {
        return null;
      }
    }
    return null;
  }
 }
 /**
 * Parse JSON response into IInvoice
 */
 function parseJsonToInvoice(response: string): IInvoice | null {
  const parsed = extractJsonFromResponse(response);
  if (!parsed) return null;
  return {
    invoice_number: extractInvoiceNumber(String(parsed.invoice_number || '')),
    invoice_date: extractDate(String(parsed.invoice_date || '')),
    vendor_name: String(parsed.vendor_name || '').replace(/\*\*/g, '').replace(/`/g, '').trim(),
    currency: extractCurrency(String(parsed.currency || '')),
    net_amount: parseAmount(parsed.net_amount as string | number),
    vat_amount: parseAmount(parsed.vat_amount as string | number),
    total_amount: parseAmount(parsed.total_amount as string | number),
  };
 }
 /**
 * Extract invoice from markdown using GPT-OSS 20B (streaming)
 */
 async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Promise<IInvoice | null> {
  const startTime = Date.now();
  console.log(`      [${queryId}] Invoice: ${markdown.length} chars, Prompt: ${JSON_EXTRACTION_PROMPT.length} chars`);
  const response = await fetch(`${OLLAMA_URL}/api/chat`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: EXTRACTION_MODEL,
      messages: [
        { role: 'user', content: 'Hi there, how are you?' },
        { role: 'assistant', content: 'Good, how can I help you today?' },
        { role: 'user', content: `Here is an invoice document:\n\n${markdown}` },
        { role: 'assistant', content: 'I have read the invoice document you provided. I can see all the text content. What would you like me to do with it?' },
        { role: 'user', content: JSON_EXTRACTION_PROMPT },
      ],
      stream: true,
      options: {
        num_ctx: 32768,  // Larger context for long invoices + thinking
        temperature: 0,  // Deterministic for JSON extraction
      },
    }),
    signal: AbortSignal.timeout(120000), // 2 min timeout
  });
  if (!response.ok) {
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] ERROR: ${response.status} (${elapsed}s)`);
    throw new Error(`Ollama API error: ${response.status}`);
  }
  // Stream the response
  let content = '';
  let thinkingContent = '';
  let thinkingStarted = false;
  let outputStarted = false;
  const reader = response.body!.getReader();
  const decoder = new TextDecoder();
  try {
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      const chunk = decoder.decode(value, { stream: true });
      for (const line of chunk.split('\n').filter(l => l.trim())) {
        try {
          const json = JSON.parse(line);
          const thinking = json.message?.thinking || '';
          if (thinking) {
            if (!thinkingStarted) {
              process.stdout.write(`      [${queryId}] THINKING: `);
              thinkingStarted = true;
            }
            process.stdout.write(thinking);
            thinkingContent += thinking;
          }
          const token = json.message?.content || '';
          if (token) {
            if (!outputStarted) {
              if (thinkingStarted) process.stdout.write('\n');
              process.stdout.write(`      [${queryId}] OUTPUT: `);
              outputStarted = true;
            }
            process.stdout.write(token);
            content += token;
          }
        } catch {
          // Ignore parse errors for partial chunks
        }
      }
    }
  } finally {
    if (thinkingStarted || outputStarted) process.stdout.write('\n');
  }
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  console.log(`      [${queryId}] Done: ${thinkingContent.length} thinking, ${content.length} output (${elapsed}s)`);
  return parseJsonToInvoice(content);
 }
 /**
 * Normalize date to YYYY-MM-DD
 */
 function normalizeDate(dateStr: string | null): string {
  if (!dateStr) return '';
  if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
  const monthMap: Record<string, string> = {
    JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
    JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
  };
  let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
  if (match) {
    return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
  }
  match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
  if (match) {
    return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
  }
  return dateStr;
 }
 /**
 * Normalize invoice number for comparison (remove spaces, lowercase)
 */
 function normalizeInvoiceNumber(s: string): string {
  return s.replace(/\s+/g, '').toLowerCase();
 }
 /**
 * Compare extracted invoice against expected
 */
 function compareInvoice(
  extracted: IInvoice,
  expected: IInvoice
 ): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  // Invoice number - normalize spaces for comparison
  const extNum = normalizeInvoiceNumber(extracted.invoice_number || '');
  const expNum = normalizeInvoiceNumber(expected.invoice_number || '');
  if (extNum !== expNum) {
    errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
    errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find test cases with existing debug markdown
 */
 function findTestCases(): ITestCase[] {
  const invoicesDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(invoicesDir)) return [];
  const testCases: ITestCase[] = [];
  for (const invoiceName of TEST_INVOICES) {
    const markdownPath = path.join(invoicesDir, `${invoiceName}.debug.md`);
    const jsonPath = path.join(invoicesDir, `${invoiceName}.json`);
    if (fs.existsSync(markdownPath) && fs.existsSync(jsonPath)) {
      testCases.push({
        name: invoiceName,
        markdownPath,
        jsonPath,
      });
    } else {
      if (!fs.existsSync(markdownPath)) {
        console.warn(`Warning: Missing markdown: ${markdownPath}`);
      }
      if (!fs.existsSync(jsonPath)) {
        console.warn(`Warning: Missing JSON: ${jsonPath}`);
      }
    }
  }
  return testCases;
 }
 // ============ TESTS ============
 const testCases = findTestCases();
 console.log(`\n========================================`);
 console.log(`  EXTRACTION TUNING TEST`);
 console.log(`  (Skips OCR, uses existing .debug.md)`);
 console.log(`========================================`);
 console.log(`  Testing ${testCases.length} invoices:`);
 for (const tc of testCases) {
  console.log(`    - ${tc.name}`);
 }
 console.log(`========================================\n`);
 tap.test('Setup Ollama + GPT-OSS 20B', async () => {
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  const extractionOk = await ensureExtractionModel();
  expect(extractionOk).toBeTrue();
 });
 let passedCount = 0;
 let failedCount = 0;
 for (const tc of testCases) {
  tap.test(`Extract ${tc.name}`, async () => {
    const expected: IInvoice = JSON.parse(fs.readFileSync(tc.jsonPath, 'utf-8'));
    const markdown = fs.readFileSync(tc.markdownPath, 'utf-8');
    console.log(`\n    ========================================`);
    console.log(`    === ${tc.name} ===`);
    console.log(`    ========================================`);
    console.log(`    EXPECTED: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
    console.log(`    Markdown: ${markdown.length} chars`);
    const startTime = Date.now();
    const extracted = await extractInvoiceFromMarkdown(markdown, tc.name);
    if (!extracted) {
      failedCount++;
      console.log(`\n    Result: ✗ FAILED TO PARSE (${((Date.now() - startTime) / 1000).toFixed(1)}s)`);
      return;
    }
    const elapsedMs = Date.now() - startTime;
    console.log(`    EXTRACTED: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`\n    Result: ✓ MATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`\n    Result: ✗ MISMATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
      console.log(`    ERRORS:`);
      result.errors.forEach(e => console.log(`      - ${e}`));
    }
  });
 }
 tap.test('Summary', async () => {
  const totalInvoices = testCases.length;
  const accuracy = totalInvoices > 0 ? (passedCount / totalInvoices) * 100 : 0;
  console.log(`\n========================================`);
  console.log(`   Extraction Tuning Summary`);
  console.log(`========================================`);
  console.log(`  Model:     ${EXTRACTION_MODEL}`);
  console.log(`  Passed:    ${passedCount}/${totalInvoices}`);
  console.log(`  Failed:    ${failedCount}/${totalInvoices}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`========================================\n`);
 });
 export default tap.start();
--- a/test/test.invoices.failed.ts
+++ b/test/test.invoices.failed.ts
@@ -0,0 +1,695 @@
 /**
 * Focused test for failed invoice extractions
 *
 * Tests only the 4 invoices that failed in the main test:
 * - consensus_2021-09: invoice_number "2021/1384" → "20211384" (slash stripped)
 * - hetzner_2022-04: model hallucinated after 281s thinking
 * - qonto_2021-08: invoice_number "08-21-INVOICE-410870" → "4108705" (prefix stripped)
 * - qonto_2021-09: invoice_number "09-21-INVOICE-4303642" → "4303642" (prefix stripped)
 *
 * Run with: tstest test/test.invoices.failed.ts --verbose
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureNanonetsOcr, ensureMiniCpm, isContainerRunning } from './helpers/docker.js';
 const NANONETS_URL = 'http://localhost:8000/v1';
 const NANONETS_MODEL = 'nanonets/Nanonets-OCR2-3B';
 const OLLAMA_URL = 'http://localhost:11434';
 const EXTRACTION_MODEL = 'gpt-oss:20b';
 // Temp directory for storing markdown between stages
 const TEMP_MD_DIR = path.join(os.tmpdir(), 'nanonets-invoices-failed-debug');
 // Only test these specific invoices that failed
 const FAILED_INVOICES = [
  'consensus_2021-09',
  'hetzner_2022-04',
  'qonto_2021-08',
  'qonto_2021-09',
 ];
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 interface IImageData {
  base64: string;
  width: number;
  height: number;
  pageNum: number;
 }
 interface ITestCase {
  name: string;
  pdfPath: string;
  jsonPath: string;
  markdownPath?: string;
 }
 // Nanonets-specific prompt for document OCR to markdown
 const NANONETS_OCR_PROMPT = `Extract the text from the above document as if you were reading it naturally.
 Return the tables in html format.
 Return the equations in LaTeX representation.
 If there is an image in the document and image caption is not present, add a small description inside <img></img> tag.
 Watermarks should be wrapped in brackets. Ex: <watermark>OFFICIAL COPY</watermark>.
 Page numbers should be wrapped in brackets. Ex: <page_number>14</page_number>.`;
 // JSON extraction prompt for GPT-OSS 20B
 const JSON_EXTRACTION_PROMPT = `You are an invoice data extractor. Below is an invoice document converted to text/markdown. Extract the key invoice fields as JSON.
 IMPORTANT RULES:
 1. invoice_number: The unique invoice/document number (NOT VAT ID, NOT customer ID). PRESERVE ALL CHARACTERS including slashes, dashes, and prefixes.
 2. invoice_date: Format as YYYY-MM-DD
 3. vendor_name: The company that issued the invoice
 4. currency: EUR, USD, or GBP
 5. net_amount: Amount before tax
 6. vat_amount: Tax/VAT amount
 7. total_amount: Final total (gross amount)
 Return ONLY this JSON format, no explanation:
 {
  "invoice_number": "INV-2024-001",
  "invoice_date": "2024-01-15",
  "vendor_name": "Company Name",
  "currency": "EUR",
  "net_amount": 100.00,
  "vat_amount": 19.00,
  "total_amount": 119.00
 }
 INVOICE TEXT:
 `;
 const PATCH_SIZE = 14;
 /**
 * Estimate visual tokens for an image based on dimensions
 */
 function estimateVisualTokens(width: number, height: number): number {
  return Math.ceil((width * height) / (PATCH_SIZE * PATCH_SIZE));
 }
 /**
 * Process images one page at a time for reliability
 */
 function batchImages(images: IImageData[]): IImageData[][] {
  return images.map(img => [img]);
 }
 /**
 * Convert PDF to JPEG images using ImageMagick with dimension tracking
 */
 function convertPdfToImages(pdfPath: string): IImageData[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.jpg');
  try {
    execSync(
      `convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f: string) => f.endsWith('.jpg')).sort();
    const images: IImageData[] = [];
    for (let i = 0; i < files.length; i++) {
      const file = files[i];
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      const dimensions = execSync(`identify -format "%w %h" "${imagePath}"`, { encoding: 'utf-8' }).trim();
      const [width, height] = dimensions.split(' ').map(Number);
      images.push({
        base64: imageData.toString('base64'),
        width,
        height,
        pageNum: i + 1,
      });
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Convert a batch of pages to markdown using Nanonets-OCR-s
 */
 async function convertBatchToMarkdown(batch: IImageData[]): Promise<string> {
  const startTime = Date.now();
  const pageNums = batch.map(img => img.pageNum).join(', ');
  const content: Array<{ type: string; image_url?: { url: string }; text?: string }> = [];
  for (const img of batch) {
    content.push({
      type: 'image_url',
      image_url: { url: `data:image/jpeg;base64,${img.base64}` },
    });
  }
  const promptText = batch.length > 1
    ? `${NANONETS_OCR_PROMPT}\n\nPlease clearly separate each page's content with "--- PAGE N ---" markers, where N is the page number starting from ${batch[0].pageNum}.`
    : NANONETS_OCR_PROMPT;
  content.push({ type: 'text', text: promptText });
  const response = await fetch(`${NANONETS_URL}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': 'Bearer dummy',
    },
    body: JSON.stringify({
      model: NANONETS_MODEL,
      messages: [{
        role: 'user',
        content,
      }],
      max_tokens: 4096 * batch.length,
      temperature: 0.0,
    }),
    signal: AbortSignal.timeout(600000),
  });
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(`Nanonets API error: ${response.status} - ${errorText}`);
  }
  const data = await response.json();
  let responseContent = (data.choices?.[0]?.message?.content || '').trim();
  if (batch.length === 1 && !responseContent.includes('--- PAGE')) {
    responseContent = `--- PAGE ${batch[0].pageNum} ---\n${responseContent}`;
  }
  console.log(`        Pages [${pageNums}]: ${responseContent.length} chars (${elapsed}s)`);
  return responseContent;
 }
 /**
 * Convert all pages of a document to markdown using smart batching
 */
 async function convertDocumentToMarkdown(images: IImageData[], docName: string): Promise<string> {
  const batches = batchImages(images);
  console.log(`      [${docName}] Processing ${images.length} page(s) in ${batches.length} batch(es)...`);
  const markdownParts: string[] = [];
  for (let i = 0; i < batches.length; i++) {
    const batch = batches[i];
    const batchTokens = batch.reduce((sum, img) => sum + estimateVisualTokens(img.width, img.height), 0);
    console.log(`        Batch ${i + 1}: ${batch.length} page(s), ~${batchTokens} tokens`);
    const markdown = await convertBatchToMarkdown(batch);
    markdownParts.push(markdown);
  }
  const fullMarkdown = markdownParts.join('\n\n');
  console.log(`      [${docName}] Complete: ${fullMarkdown.length} chars total`);
  return fullMarkdown;
 }
 /**
 * Stop Nanonets container
 */
 function stopNanonets(): void {
  console.log('    [Docker] Stopping Nanonets container...');
  try {
    execSync('docker stop nanonets-test 2>/dev/null || true', { stdio: 'pipe' });
    execSync('sleep 5', { stdio: 'pipe' });
    console.log('    [Docker] Nanonets stopped');
  } catch {
    console.log('    [Docker] Nanonets was not running');
  }
 }
 /**
 * Ensure GPT-OSS 20B model is available
 */
 async function ensureExtractionModel(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === EXTRACTION_MODEL)) {
        console.log(`    [Ollama] Model available: ${EXTRACTION_MODEL}`);
        return true;
      }
    }
  } catch {
    return false;
  }
  console.log(`    [Ollama] Pulling ${EXTRACTION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: EXTRACTION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 /**
 * Parse amount from string (handles European format)
 */
 function parseAmount(s: string | number | undefined): number {
  if (s === undefined || s === null) return 0;
  if (typeof s === 'number') return s;
  const match = s.match(/([\d.,]+)/);
  if (!match) return 0;
  const numStr = match[1];
  const normalized = numStr.includes(',') && numStr.indexOf(',') > numStr.lastIndexOf('.')
    ? numStr.replace(/\./g, '').replace(',', '.')
    : numStr.replace(/,/g, '');
  return parseFloat(normalized) || 0;
 }
 /**
 * Extract invoice number - MINIMAL normalization for debugging
 */
 function extractInvoiceNumber(s: string | undefined): string {
  if (!s) return '';
  // Only remove markdown formatting, preserve everything else
  return s.replace(/\*\*/g, '').replace(/`/g, '').trim();
 }
 /**
 * Extract date (YYYY-MM-DD) from response
 */
 function extractDate(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const isoMatch = clean.match(/(\d{4}-\d{2}-\d{2})/);
  if (isoMatch) return isoMatch[1];
  const dmyMatch = clean.match(/(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})/);
  if (dmyMatch) {
    return `${dmyMatch[3]}-${dmyMatch[2].padStart(2, '0')}-${dmyMatch[1].padStart(2, '0')}`;
  }
  return clean.replace(/[^\d-]/g, '').trim();
 }
 /**
 * Extract currency
 */
 function extractCurrency(s: string | undefined): string {
  if (!s) return 'EUR';
  const upper = s.toUpperCase();
  if (upper.includes('EUR') || upper.includes('€')) return 'EUR';
  if (upper.includes('USD') || upper.includes('$')) return 'USD';
  if (upper.includes('GBP') || upper.includes('£')) return 'GBP';
  return 'EUR';
 }
 /**
 * Extract JSON from response
 */
 function extractJsonFromResponse(response: string): Record<string, unknown> | null {
  let cleanResponse = response.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  const codeBlockMatch = cleanResponse.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : cleanResponse;
  try {
    return JSON.parse(jsonStr);
  } catch {
    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      try {
        return JSON.parse(jsonMatch[0]);
      } catch {
        return null;
      }
    }
    return null;
  }
 }
 /**
 * Parse JSON response into IInvoice
 */
 function parseJsonToInvoice(response: string): IInvoice | null {
  const parsed = extractJsonFromResponse(response);
  if (!parsed) return null;
  return {
    invoice_number: extractInvoiceNumber(String(parsed.invoice_number || '')),
    invoice_date: extractDate(String(parsed.invoice_date || '')),
    vendor_name: String(parsed.vendor_name || '').replace(/\*\*/g, '').replace(/`/g, '').trim(),
    currency: extractCurrency(String(parsed.currency || '')),
    net_amount: parseAmount(parsed.net_amount as string | number),
    vat_amount: parseAmount(parsed.vat_amount as string | number),
    total_amount: parseAmount(parsed.total_amount as string | number),
  };
 }
 /**
 * Extract invoice from markdown using GPT-OSS 20B (streaming)
 */
 async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Promise<IInvoice | null> {
  const startTime = Date.now();
  const fullPrompt = JSON_EXTRACTION_PROMPT + markdown;
  // Log exact prompt
  console.log(`\n      [${queryId}] ===== PROMPT =====`);
  console.log(fullPrompt);
  console.log(`      [${queryId}] ===== END PROMPT (${fullPrompt.length} chars) =====\n`);
  const response = await fetch(`${OLLAMA_URL}/api/chat`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: EXTRACTION_MODEL,
      messages: [
        { role: 'user', content: 'Hi there, how are you?' },
        { role: 'assistant', content: 'Good, how can I help you today?' },
        { role: 'user', content: fullPrompt },
      ],
      stream: true,
    }),
    signal: AbortSignal.timeout(600000),
  });
  if (!response.ok) {
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] ERROR: ${response.status} (${elapsed}s)`);
    throw new Error(`Ollama API error: ${response.status}`);
  }
  // Stream the response
  let content = '';
  let thinkingContent = '';
  let thinkingStarted = false;
  let outputStarted = false;
  const reader = response.body!.getReader();
  const decoder = new TextDecoder();
  try {
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      const chunk = decoder.decode(value, { stream: true });
      for (const line of chunk.split('\n').filter(l => l.trim())) {
        try {
          const json = JSON.parse(line);
          const thinking = json.message?.thinking || '';
          if (thinking) {
            if (!thinkingStarted) {
              process.stdout.write(`      [${queryId}] THINKING: `);
              thinkingStarted = true;
            }
            process.stdout.write(thinking);
            thinkingContent += thinking;
          }
          const token = json.message?.content || '';
          if (token) {
            if (!outputStarted) {
              if (thinkingStarted) process.stdout.write('\n');
              process.stdout.write(`      [${queryId}] OUTPUT: `);
              outputStarted = true;
            }
            process.stdout.write(token);
            content += token;
          }
        } catch {
          // Ignore parse errors for partial chunks
        }
      }
    }
  } finally {
    if (thinkingStarted || outputStarted) process.stdout.write('\n');
  }
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  console.log(`      [${queryId}] Done: ${thinkingContent.length} thinking chars, ${content.length} output chars (${elapsed}s)`);
  // Log raw response for debugging
  console.log(`      [${queryId}] RAW RESPONSE: ${content}`);
  return parseJsonToInvoice(content);
 }
 /**
 * Extract invoice (single pass)
 */
 async function extractInvoice(markdown: string, docName: string): Promise<IInvoice> {
  console.log(`    [${docName}] Extracting...`);
  const invoice = await extractInvoiceFromMarkdown(markdown, docName);
  if (!invoice) {
    return {
      invoice_number: '',
      invoice_date: '',
      vendor_name: '',
      currency: 'EUR',
      net_amount: 0,
      vat_amount: 0,
      total_amount: 0,
    };
  }
  console.log(`    [${docName}] Extracted: ${JSON.stringify(invoice, null, 2)}`);
  return invoice;
 }
 /**
 * Normalize date to YYYY-MM-DD
 */
 function normalizeDate(dateStr: string | null): string {
  if (!dateStr) return '';
  if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
  const monthMap: Record<string, string> = {
    JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
    JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
  };
  let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
  if (match) {
    return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
  }
  match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
  if (match) {
    return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
  }
  return dateStr;
 }
 /**
 * Compare extracted invoice against expected - detailed output
 */
 function compareInvoice(
  extracted: IInvoice,
  expected: IInvoice
 ): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  // Invoice number comparison - exact match after whitespace normalization
  const extNum = extracted.invoice_number?.trim() || '';
  const expNum = expected.invoice_number?.trim() || '';
  if (extNum.toLowerCase() !== expNum.toLowerCase()) {
    errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
    errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find test cases for failed invoices only
 */
 function findTestCases(): ITestCase[] {
  const testDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(testDir)) return [];
  const files = fs.readdirSync(testDir);
  const testCases: ITestCase[] = [];
  for (const invoiceName of FAILED_INVOICES) {
    const pdfFile = `${invoiceName}.pdf`;
    const jsonFile = `${invoiceName}.json`;
    if (files.includes(pdfFile) && files.includes(jsonFile)) {
      testCases.push({
        name: invoiceName,
        pdfPath: path.join(testDir, pdfFile),
        jsonPath: path.join(testDir, jsonFile),
      });
    } else {
      console.warn(`Warning: Missing files for ${invoiceName}`);
    }
  }
  return testCases;
 }
 // ============ TESTS ============
 const testCases = findTestCases();
 console.log(`\n========================================`);
 console.log(`  FAILED INVOICES DEBUG TEST`);
 console.log(`========================================`);
 console.log(`  Testing ${testCases.length} failed invoices:`);
 for (const tc of testCases) {
  console.log(`    - ${tc.name}`);
 }
 console.log(`========================================\n`);
 // Ensure temp directory exists
 if (!fs.existsSync(TEMP_MD_DIR)) {
  fs.mkdirSync(TEMP_MD_DIR, { recursive: true });
 }
 // -------- STAGE 1: OCR with Nanonets --------
 tap.test('Stage 1: Setup Nanonets', async () => {
  console.log('\n========== STAGE 1: Nanonets OCR ==========\n');
  const ok = await ensureNanonetsOcr();
  expect(ok).toBeTrue();
 });
 tap.test('Stage 1: Convert failed invoices to markdown', async () => {
  console.log('\n  Converting failed invoice PDFs to markdown with Nanonets-OCR-s...\n');
  for (const tc of testCases) {
    console.log(`\n    === ${tc.name} ===`);
    const images = convertPdfToImages(tc.pdfPath);
    console.log(`      Pages: ${images.length}`);
    const markdown = await convertDocumentToMarkdown(images, tc.name);
    const mdPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
    fs.writeFileSync(mdPath, markdown);
    tc.markdownPath = mdPath;
    console.log(`      Saved: ${mdPath}`);
    // Also save to .nogit for inspection
    const debugMdPath = path.join(process.cwd(), '.nogit/invoices', `${tc.name}.debug.md`);
    fs.writeFileSync(debugMdPath, markdown);
    console.log(`      Debug copy: ${debugMdPath}`);
  }
  console.log('\n  Stage 1 complete: All failed invoices converted to markdown\n');
 });
 tap.test('Stage 1: Stop Nanonets', async () => {
  stopNanonets();
  await new Promise(resolve => setTimeout(resolve, 3000));
  expect(isContainerRunning('nanonets-test')).toBeFalse();
 });
 // -------- STAGE 2: Extraction with GPT-OSS 20B --------
 tap.test('Stage 2: Setup Ollama + GPT-OSS 20B', async () => {
  console.log('\n========== STAGE 2: GPT-OSS 20B Extraction ==========\n');
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  const extractionOk = await ensureExtractionModel();
  expect(extractionOk).toBeTrue();
 });
 let passedCount = 0;
 let failedCount = 0;
 for (const tc of testCases) {
  tap.test(`Stage 2: Extract ${tc.name}`, async () => {
    const expected: IInvoice = JSON.parse(fs.readFileSync(tc.jsonPath, 'utf-8'));
    console.log(`\n    ========================================`);
    console.log(`    === ${tc.name} ===`);
    console.log(`    ========================================`);
    console.log(`    EXPECTED:`);
    console.log(`      invoice_number: "${expected.invoice_number}"`);
    console.log(`      invoice_date:   "${expected.invoice_date}"`);
    console.log(`      vendor_name:    "${expected.vendor_name}"`);
    console.log(`      total_amount:   ${expected.total_amount} ${expected.currency}`);
    const startTime = Date.now();
    const mdPath = path.join(TEMP_MD_DIR, `${tc.name}.md`);
    if (!fs.existsSync(mdPath)) {
      throw new Error(`Markdown not found: ${mdPath}. Run Stage 1 first.`);
    }
    const markdown = fs.readFileSync(mdPath, 'utf-8');
    console.log(`    Markdown: ${markdown.length} chars`);
    const extracted = await extractInvoice(markdown, tc.name);
    const elapsedMs = Date.now() - startTime;
    console.log(`\n    EXTRACTED:`);
    console.log(`      invoice_number: "${extracted.invoice_number}"`);
    console.log(`      invoice_date:   "${extracted.invoice_date}"`);
    console.log(`      vendor_name:    "${extracted.vendor_name}"`);
    console.log(`      total_amount:   ${extracted.total_amount} ${extracted.currency}`);
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`\n    Result: ✓ MATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`\n    Result: ✗ MISMATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
      console.log(`    ERRORS:`);
      result.errors.forEach(e => console.log(`      - ${e}`));
    }
    // Don't fail the test - we're debugging
    // expect(result.match).toBeTrue();
  });
 }
 tap.test('Summary', async () => {
  const totalInvoices = testCases.length;
  const accuracy = totalInvoices > 0 ? (passedCount / totalInvoices) * 100 : 0;
  console.log(`\n========================================`);
  console.log(`   Failed Invoices Debug Summary`);
  console.log(`========================================`);
  console.log(`  Passed:    ${passedCount}/${totalInvoices}`);
  console.log(`  Failed:    ${failedCount}/${totalInvoices}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`========================================`);
  console.log(`  Markdown files saved to: ${TEMP_MD_DIR}`);
  console.log(`  Debug copies in: .nogit/invoices/*.debug.md`);
  console.log(`========================================\n`);
  // Don't cleanup temp files for debugging
  console.log(`  Keeping temp files for debugging.\n`);
 });
 export default tap.start();
--- a/test/test.invoices.minicpm.ts
+++ b/test/test.invoices.minicpm.ts
@@ -0,0 +1,488 @@
 /**
 * Invoice extraction test using MiniCPM-V via smartagent DualAgentOrchestrator
 *
 * Uses vision-capable orchestrator with JsonValidatorTool for self-validation:
 * 1. Pass images to the orchestrator
 * 2. Driver extracts invoice data and validates JSON before completing
 * 3. If validation fails, driver retries within the same task
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureMiniCpm } from './helpers/docker.js';
 import { SmartAi } from '@push.rocks/smartai';
 import { DualAgentOrchestrator, JsonValidatorTool } from '@push.rocks/smartagent';
 const OLLAMA_URL = 'http://localhost:11434';
 const MODEL = 'openbmb/minicpm-v4.5:q8_0';
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 // SmartAi instance and orchestrator (initialized in setup)
 let smartAi: SmartAi;
 let orchestrator: DualAgentOrchestrator;
 /**
 * Convert PDF to PNG images using ImageMagick
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    execSync(
      `convert -density 300 -quality 95 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 const EXTRACTION_PROMPT = `Extract invoice data from the provided image(s).
 IMPORTANT: You must output a valid JSON object with these exact fields:
 {
  "invoice_number": "the invoice number (not VAT ID, not customer ID)",
  "invoice_date": "YYYY-MM-DD format",
  "vendor_name": "company that issued the invoice",
  "currency": "EUR, USD, or GBP",
  "net_amount": 0.00,
  "vat_amount": 0.00,
  "total_amount": 0.00
 }
 Before completing, use the json.validate tool to verify your output is valid JSON with all required fields.
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON_HERE", "requiredFields": ["invoice_number", "invoice_date", "vendor_name", "currency", "net_amount", "vat_amount", "total_amount"]}</params>
 </tool_call>
 Only complete the task after validation passes. Output the final JSON in <task_complete> tags.`;
 /**
 * Parse amount from string (handles European format)
 */
 function parseAmount(s: string | number | undefined): number {
  if (s === undefined || s === null) return 0;
  if (typeof s === 'number') return s;
  const match = s.match(/([\d.,]+)/);
  if (!match) return 0;
  const numStr = match[1];
  // Handle European format: 1.234,56 → 1234.56
  const normalized = numStr.includes(',') && numStr.indexOf(',') > numStr.lastIndexOf('.')
    ? numStr.replace(/\./g, '').replace(',', '.')
    : numStr.replace(/,/g, '');
  return parseFloat(normalized) || 0;
 }
 /**
 * Extract invoice number from potentially verbose response
 */
 function extractInvoiceNumber(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const patterns = [
    /\b([A-Z]{2,3}\d{10,})\b/i,           // IEE2022006460244
    /\b([A-Z]\d{8,})\b/i,                  // R0014359508
    /\b(INV[-\s]?\d{4}[-\s]?\d+)\b/i,     // INV-2024-001
    /\b(\d{7,})\b/,                        // 1579087430
  ];
  for (const pattern of patterns) {
    const match = clean.match(pattern);
    if (match) return match[1];
  }
  return clean.replace(/[^A-Z0-9-]/gi, '').trim() || clean;
 }
 /**
 * Extract date (YYYY-MM-DD) from response
 */
 function extractDate(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const isoMatch = clean.match(/(\d{4}-\d{2}-\d{2})/);
  if (isoMatch) return isoMatch[1];
  // Try DD/MM/YYYY or DD.MM.YYYY
  const dmyMatch = clean.match(/(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})/);
  if (dmyMatch) {
    return `${dmyMatch[3]}-${dmyMatch[2].padStart(2, '0')}-${dmyMatch[1].padStart(2, '0')}`;
  }
  return clean.replace(/[^\d-]/g, '').trim();
 }
 /**
 * Extract currency
 */
 function extractCurrency(s: string | undefined): string {
  if (!s) return 'EUR';
  const upper = s.toUpperCase();
  if (upper.includes('EUR') || upper.includes('€')) return 'EUR';
  if (upper.includes('USD') || upper.includes('$')) return 'USD';
  if (upper.includes('GBP') || upper.includes('£')) return 'GBP';
  return 'EUR';
 }
 /**
 * Extract JSON from response (handles markdown code blocks and task_complete tags)
 */
 function extractJsonFromResponse(response: string): Record<string, unknown> | null {
  // Try to find JSON in task_complete tags
  const completeMatch = response.match(/<task_complete>([\s\S]*?)<\/task_complete>/);
  if (completeMatch) {
    const content = completeMatch[1].trim();
    // Try to find JSON in the content
    const codeBlockMatch = content.match(/```(?:json)?\s*([\s\S]*?)```/);
    const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : content;
    try {
      return JSON.parse(jsonStr);
    } catch {
      // Try to find JSON object pattern
      const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
      if (jsonMatch) {
        try {
          return JSON.parse(jsonMatch[0]);
        } catch {
          return null;
        }
      }
    }
  }
  // Try to find JSON in markdown code block
  const codeBlockMatch = response.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : response.trim();
  try {
    return JSON.parse(jsonStr);
  } catch {
    // Try to find JSON object pattern
    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      try {
        return JSON.parse(jsonMatch[0]);
      } catch {
        return null;
      }
    }
    return null;
  }
 }
 /**
 * Parse JSON response into IInvoice
 */
 function parseJsonToInvoice(response: string): IInvoice | null {
  const parsed = extractJsonFromResponse(response);
  if (!parsed) return null;
  return {
    invoice_number: extractInvoiceNumber(String(parsed.invoice_number || '')),
    invoice_date: extractDate(String(parsed.invoice_date || '')),
    vendor_name: String(parsed.vendor_name || '').replace(/\*\*/g, '').replace(/`/g, '').trim(),
    currency: extractCurrency(String(parsed.currency || '')),
    net_amount: parseAmount(parsed.net_amount as string | number),
    vat_amount: parseAmount(parsed.vat_amount as string | number),
    total_amount: parseAmount(parsed.total_amount as string | number),
  };
 }
 /**
 * Extract invoice data using smartagent orchestrator with vision
 */
 async function extractInvoiceFromImages(images: string[]): Promise<IInvoice> {
  console.log(`    [Vision] Processing ${images.length} page(s) with smartagent DualAgentOrchestrator`);
  const startTime = Date.now();
  const result = await orchestrator.run(EXTRACTION_PROMPT, { images });
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  console.log(`    [Vision] Completed in ${elapsed}s (${result.iterations} iterations, status: ${result.status})`);
  const invoice = parseJsonToInvoice(result.result);
  if (invoice) {
    console.log(`    [Result] ${invoice.invoice_number} | ${invoice.invoice_date} | ${invoice.total_amount} ${invoice.currency}`);
    return invoice;
  }
  // Return empty invoice if parsing failed
  console.log(`    [Result] Parsing failed, returning empty invoice`);
  return {
    invoice_number: '',
    invoice_date: '',
    vendor_name: '',
    currency: 'EUR',
    net_amount: 0,
    vat_amount: 0,
    total_amount: 0,
  };
 }
 /**
 * Normalize date to YYYY-MM-DD
 */
 function normalizeDate(dateStr: string | null): string {
  if (!dateStr) return '';
  if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
  const monthMap: Record<string, string> = {
    JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
    JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
  };
  let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
  if (match) {
    return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
  }
  match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
  if (match) {
    return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
  }
  return dateStr;
 }
 /**
 * Compare extracted invoice against expected
 */
 function compareInvoice(
  extracted: IInvoice,
  expected: IInvoice
 ): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  // Compare invoice number (normalize by removing spaces and case)
  const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  if (extNum !== expNum) {
    errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  // Compare date
  if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
    errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  // Compare total amount (with tolerance)
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  // Compare currency
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find all test cases (PDF + JSON pairs) in .nogit/invoices/
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(testDir)) {
    return [];
  }
  const files = fs.readdirSync(testDir);
  const pdfFiles = files.filter((f) => f.endsWith('.pdf'));
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of pdfFiles) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  testCases.sort((a, b) => a.name.localeCompare(b.name));
  return testCases;
 }
 // Tests
 tap.test('setup: ensure Docker containers are running', async () => {
  console.log('\n[Setup] Checking Docker containers...\n');
  const minicpmOk = await ensureMiniCpm();
  expect(minicpmOk).toBeTrue();
  console.log('\n[Setup] All containers ready!\n');
 });
 tap.test('setup: initialize smartagent orchestrator', async () => {
  console.log('[Setup] Initializing SmartAi and DualAgentOrchestrator...');
  smartAi = new SmartAi({
    ollama: {
      baseUrl: OLLAMA_URL,
      model: MODEL,
      defaultOptions: {
        num_ctx: 32768,
        temperature: 0.1,
      },
      defaultTimeout: 300000, // 5 minutes for vision tasks
    },
  });
  await smartAi.start();
  orchestrator = new DualAgentOrchestrator({
    smartAiInstance: smartAi,
    defaultProvider: 'ollama',
    guardianPolicyPrompt: `You are a Guardian agent overseeing invoice extraction tasks.
 APPROVE all tool calls that:
 - Use the json.validate action to verify JSON output
 - Are reasonable attempts to complete the extraction task
 REJECT tool calls that:
 - Attempt to access external resources
 - Try to execute arbitrary code
 - Are clearly unrelated to invoice extraction`,
    driverSystemMessage: `You are an AI assistant that extracts invoice data from images.
 Your task is to analyze invoice images and extract structured data.
 You have access to a json.validate tool to verify your JSON output.
 IMPORTANT: Always validate your JSON before completing the task.
 ## Tool Usage Format
 When you need to validate JSON, output:
 <tool_call>
  <tool>json</tool>
  <action>validate</action>
  <params>{"jsonString": "YOUR_JSON", "requiredFields": ["invoice_number", "invoice_date", "vendor_name", "currency", "net_amount", "vat_amount", "total_amount"]}</params>
 </tool_call>
 ## Completion Format
 After validation passes, complete the task:
 <task_complete>
 {"invoice_number": "...", "invoice_date": "YYYY-MM-DD", ...}
 </task_complete>`,
    maxIterations: 5,
    maxConsecutiveRejections: 3,
    onToken: (token, source) => {
      if (source === 'driver') {
        process.stdout.write(token);
      }
    },
    onProgress: (event) => {
      if (event.logLevel === 'error') {
        console.error(event.logMessage);
      }
    },
  });
  // Register the JsonValidatorTool
  orchestrator.registerTool(new JsonValidatorTool());
  await orchestrator.start();
  console.log('[Setup] Orchestrator initialized!\n');
 });
 tap.test('should have MiniCPM-V model loaded', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  const data = await response.json();
  const modelNames = data.models.map((m: { name: string }) => m.name);
  expect(modelNames.some((name: string) => name.includes('minicpm'))).toBeTrue();
 });
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} invoice test cases (smartagent + MiniCPM-V)\n`);
 let passedCount = 0;
 let failedCount = 0;
 const processingTimes: number[] = [];
 for (const testCase of testCases) {
  tap.test(`should extract invoice: ${testCase.name}`, async () => {
    const expected: IInvoice = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
    const startTime = Date.now();
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`  Pages: ${images.length}`);
    const extracted = await extractInvoiceFromImages(images);
    console.log(`  Extracted: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
    const elapsedMs = Date.now() - startTime;
    processingTimes.push(elapsedMs);
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`  Result: MATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`  Result: MISMATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
      result.errors.forEach((e) => console.log(`    - ${e}`));
    }
    expect(result.match).toBeTrue();
  });
 }
 tap.test('cleanup: stop orchestrator', async () => {
  if (orchestrator) {
    await orchestrator.stop();
  }
  console.log('[Cleanup] Orchestrator stopped');
 });
 tap.test('summary', async () => {
  const totalInvoices = testCases.length;
  const accuracy = totalInvoices > 0 ? (passedCount / totalInvoices) * 100 : 0;
  const totalTimeMs = processingTimes.reduce((a, b) => a + b, 0);
  const avgTimeSec = processingTimes.length > 0 ? totalTimeMs / processingTimes.length / 1000 : 0;
  console.log(`\n========================================`);
  console.log(`   Invoice Extraction Summary`);
  console.log(`   (smartagent + ${MODEL})`);
  console.log(`========================================`);
  console.log(`  Method:    DualAgentOrchestrator with vision`);
  console.log(`  Passed:    ${passedCount}/${totalInvoices}`);
  console.log(`  Failed:    ${failedCount}/${totalInvoices}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`----------------------------------------`);
  console.log(`  Total time:   ${(totalTimeMs / 1000).toFixed(1)}s`);
  console.log(`  Avg per inv:  ${avgTimeSec.toFixed(1)}s`);
  console.log(`========================================\n`);
 });
 export default tap.start();
--- a/test/test.invoices.nanonets.ts
+++ b/test/test.invoices.nanonets.ts
@@ -0,0 +1,752 @@
 /**
 * Invoice extraction using Nanonets-OCR2-3B + GPT-OSS 20B (sequential two-stage pipeline)
 *
 * Stage 1: Nanonets-OCR2-3B converts ALL document pages to markdown (stop after completion)
 * Stage 2: GPT-OSS 20B extracts structured JSON from saved markdown (after Nanonets stops)
 *
 * This approach avoids GPU contention by running services sequentially.
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureNanonetsOcr, ensureMiniCpm, isContainerRunning } from './helpers/docker.js';
 import { SmartAi } from '@push.rocks/smartai';
 import { DualAgentOrchestrator, JsonValidatorTool } from '@push.rocks/smartagent';
 const NANONETS_URL = 'http://localhost:8000/v1';
 const NANONETS_MODEL = 'nanonets/Nanonets-OCR2-3B';
 const OLLAMA_URL = 'http://localhost:11434';
 const EXTRACTION_MODEL = 'gpt-oss:20b';
 // Persistent cache directory for storing markdown between runs
 const MD_CACHE_DIR = path.join(process.cwd(), '.nogit/invoices-md');
 // SmartAi instance for Ollama with optimized settings
 const smartAi = new SmartAi({
  ollama: {
    baseUrl: OLLAMA_URL,
    model: EXTRACTION_MODEL,
    defaultOptions: {
      num_ctx: 65536,    // 64K context for long invoices + reasoning chains
      temperature: 0,    // Deterministic for JSON extraction
      repeat_penalty: 1.3, // Penalty to prevent repetition loops
      think: true,       // Enable thinking mode for GPT-OSS reasoning
    },
    defaultTimeout: 600000,  // 10 minute timeout for large documents
  },
 });
 // DualAgentOrchestrator for structured task execution
 let orchestrator: DualAgentOrchestrator;
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 interface IImageData {
  base64: string;
  width: number;
  height: number;
  pageNum: number;
 }
 interface ITestCase {
  name: string;
  pdfPath: string;
  jsonPath: string;
  markdownPath?: string;
 }
 // Nanonets-specific prompt for document OCR to markdown
 const NANONETS_OCR_PROMPT = `Extract the text from the above document as if you were reading it naturally.
 Return the tables in html format.
 Return the equations in LaTeX representation.
 If there is an image in the document and image caption is not present, add a small description inside <img></img> tag.
 Watermarks should be wrapped in brackets. Ex: <watermark>OFFICIAL COPY</watermark>.
 Page numbers should be wrapped in brackets. Ex: <page_number>14</page_number>.`;
 // JSON extraction prompt for GPT-OSS 20B (sent AFTER the invoice text is provided)
 const JSON_EXTRACTION_PROMPT = `Extract key fields from the invoice. Return ONLY valid JSON.
 WHERE TO FIND DATA:
 - invoice_number, invoice_date, vendor_name: Look in the HEADER section at the TOP of PAGE 1 (near "Invoice no.", "Invoice date:", "Rechnungsnummer"). Use common sense. Btw. an invoice number might start on INV* .
 - net_amount, vat_amount, total_amount: Look in the SUMMARY section at the BOTTOM (look for "Total", "Amount due", "Gesamtbetrag")
 RULES:
 1. Use common sense.
 2. invoice_date: Convert to YYYY-MM-DD format (e.g., "14/04/2022" → "2022-04-14")
 3. vendor_name: The company issuing the invoice
 4. currency: EUR, USD, or GBP
 5. net_amount: Total before tax
 6. vat_amount: Tax amount
 7. total_amount: Final total with tax
 JSON only:
 {"invoice_number":"X","invoice_date":"YYYY-MM-DD","vendor_name":"X","currency":"EUR","net_amount":0,"vat_amount":0,"total_amount":0}
 Double check for valid JSON syntax.
 `;
 // Constants for smart batching
 const PATCH_SIZE = 14;  // Qwen2.5-VL uses 14x14 patches
 /**
 * Estimate visual tokens for an image based on dimensions
 */
 function estimateVisualTokens(width: number, height: number): number {
  return Math.ceil((width * height) / (PATCH_SIZE * PATCH_SIZE));
 }
 /**
 * Process images one page at a time for reliability
 */
 function batchImages(images: IImageData[]): IImageData[][] {
  // One page per batch for reliable processing
  return images.map(img => [img]);
 }
 /**
 * Convert PDF to JPEG images using ImageMagick with dimension tracking
 */
 function convertPdfToImages(pdfPath: string): IImageData[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.jpg');
  try {
    execSync(
      `convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f: string) => f.endsWith('.jpg')).sort();
    const images: IImageData[] = [];
    for (let i = 0; i < files.length; i++) {
      const file = files[i];
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      // Get image dimensions using identify command
      const dimensions = execSync(`identify -format "%w %h" "${imagePath}"`, { encoding: 'utf-8' }).trim();
      const [width, height] = dimensions.split(' ').map(Number);
      images.push({
        base64: imageData.toString('base64'),
        width,
        height,
        pageNum: i + 1,
      });
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Convert a batch of pages to markdown using Nanonets-OCR-s
 */
 async function convertBatchToMarkdown(batch: IImageData[]): Promise<string> {
  const startTime = Date.now();
  const pageNums = batch.map(img => img.pageNum).join(', ');
  // Build content array with all images first, then the prompt
  const content: Array<{ type: string; image_url?: { url: string }; text?: string }> = [];
  for (const img of batch) {
    content.push({
      type: 'image_url',
      image_url: { url: `data:image/jpeg;base64,${img.base64}` },
    });
  }
  // Add prompt with page separator instruction if multiple pages
  const promptText = batch.length > 1
    ? `${NANONETS_OCR_PROMPT}\n\nPlease clearly separate each page's content with "--- PAGE N ---" markers, where N is the page number starting from ${batch[0].pageNum}.`
    : NANONETS_OCR_PROMPT;
  content.push({ type: 'text', text: promptText });
  const response = await fetch(`${NANONETS_URL}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': 'Bearer dummy',
    },
    body: JSON.stringify({
      model: NANONETS_MODEL,
      messages: [{
        role: 'user',
        content,
      }],
      max_tokens: 4096 * batch.length,  // Scale output tokens with batch size
      temperature: 0.0,
    }),
    signal: AbortSignal.timeout(600000),  // 10 minute timeout for OCR
  });
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(`Nanonets API error: ${response.status} - ${errorText}`);
  }
  const data = await response.json();
  let responseContent = (data.choices?.[0]?.message?.content || '').trim();
  // For single-page batches, add page marker if not present
  if (batch.length === 1 && !responseContent.includes('--- PAGE')) {
    responseContent = `--- PAGE ${batch[0].pageNum} ---\n${responseContent}`;
  }
  console.log(`        Pages [${pageNums}]: ${responseContent.length} chars (${elapsed}s)`);
  return responseContent;
 }
 /**
 * Convert all pages of a document to markdown using smart batching
 */
 async function convertDocumentToMarkdown(images: IImageData[], docName: string): Promise<string> {
  const batches = batchImages(images);
  console.log(`      [${docName}] Processing ${images.length} page(s) in ${batches.length} batch(es)...`);
  const markdownParts: string[] = [];
  for (let i = 0; i < batches.length; i++) {
    const batch = batches[i];
    const batchTokens = batch.reduce((sum, img) => sum + estimateVisualTokens(img.width, img.height), 0);
    console.log(`        Batch ${i + 1}: ${batch.length} page(s), ~${batchTokens} tokens`);
    const markdown = await convertBatchToMarkdown(batch);
    markdownParts.push(markdown);
  }
  const fullMarkdown = markdownParts.join('\n\n');
  console.log(`      [${docName}] Complete: ${fullMarkdown.length} chars total`);
  return fullMarkdown;
 }
 /**
 * Stop Nanonets container
 */
 function stopNanonets(): void {
  console.log('    [Docker] Stopping Nanonets container...');
  try {
    execSync('docker stop nanonets-test 2>/dev/null || true', { stdio: 'pipe' });
    execSync('sleep 5', { stdio: 'pipe' });
    console.log('    [Docker] Nanonets stopped');
  } catch {
    console.log('    [Docker] Nanonets was not running');
  }
 }
 /**
 * Ensure GPT-OSS 20B model is available
 */
 async function ensureExtractionModel(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === EXTRACTION_MODEL)) {
        console.log(`    [Ollama] Model available: ${EXTRACTION_MODEL}`);
        return true;
      }
    }
  } catch {
    return false;
  }
  console.log(`    [Ollama] Pulling ${EXTRACTION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: EXTRACTION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 /**
 * Parse amount from string (handles European format)
 */
 function parseAmount(s: string | number | undefined): number {
  if (s === undefined || s === null) return 0;
  if (typeof s === 'number') return s;
  const match = s.match(/([\d.,]+)/);
  if (!match) return 0;
  const numStr = match[1];
  const normalized = numStr.includes(',') && numStr.indexOf(',') > numStr.lastIndexOf('.')
    ? numStr.replace(/\./g, '').replace(',', '.')
    : numStr.replace(/,/g, '');
  return parseFloat(normalized) || 0;
 }
 /**
 * Extract invoice number from potentially verbose response
 */
 function extractInvoiceNumber(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const patterns = [
    /\b([A-Z]{2,3}\d{10,})\b/i,
    /\b([A-Z]\d{8,})\b/i,
    /\b(INV[-\s]?\d{4}[-\s]?\d+)\b/i,
    /\b(\d{7,})\b/,
  ];
  for (const pattern of patterns) {
    const match = clean.match(pattern);
    if (match) return match[1];
  }
  return clean.replace(/[^A-Z0-9-]/gi, '').trim() || clean;
 }
 /**
 * Extract date (YYYY-MM-DD) from response
 */
 function extractDate(s: string | undefined): string {
  if (!s) return '';
  let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
  const isoMatch = clean.match(/(\d{4}-\d{2}-\d{2})/);
  if (isoMatch) return isoMatch[1];
  const dmyMatch = clean.match(/(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})/);
  if (dmyMatch) {
    return `${dmyMatch[3]}-${dmyMatch[2].padStart(2, '0')}-${dmyMatch[1].padStart(2, '0')}`;
  }
  return clean.replace(/[^\d-]/g, '').trim();
 }
 /**
 * Extract currency
 */
 function extractCurrency(s: string | undefined): string {
  if (!s) return 'EUR';
  const upper = s.toUpperCase();
  if (upper.includes('EUR') || upper.includes('€')) return 'EUR';
  if (upper.includes('USD') || upper.includes('$')) return 'USD';
  if (upper.includes('GBP') || upper.includes('£')) return 'GBP';
  return 'EUR';
 }
 /**
 * Try to extract valid JSON from a response string
 */
 function tryExtractJson(response: string): Record<string, unknown> | null {
  // Remove thinking tags
  let clean = response.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
  // Try code block
  const codeBlockMatch = clean.match(/```(?:json)?\s*([\s\S]*?)```/);
  const jsonStr = codeBlockMatch ? codeBlockMatch[1].trim() : clean;
  try {
    return JSON.parse(jsonStr);
  } catch {
    // Try to find JSON object
    const jsonMatch = jsonStr.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      try {
        return JSON.parse(jsonMatch[0]);
      } catch {
        return null;
      }
    }
    return null;
  }
 }
 /**
 * Extract invoice from markdown using smartagent DualAgentOrchestrator
 * Validates JSON and retries if invalid
 */
 async function extractInvoiceFromMarkdown(markdown: string, queryId: string): Promise<IInvoice | null> {
  const startTime = Date.now();
  const maxRetries = 2;
  console.log(`      [${queryId}] Invoice: ${markdown.length} chars`);
  // Build the extraction task with document context
  const taskPrompt = `Extract the invoice data from this document and output ONLY the JSON:
 ${markdown}
 ${JSON_EXTRACTION_PROMPT}`;
  try {
    let result = await orchestrator.run(taskPrompt);
    let elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] Status: ${result.status}, Iterations: ${result.iterations} (${elapsed}s)`);
    // Try to parse JSON from result
    let jsonData: Record<string, unknown> | null = null;
    let responseText = result.result || '';
    if (result.success && responseText) {
      jsonData = tryExtractJson(responseText);
    }
    // Fallback: try parsing from history
    if (!jsonData && result.history?.length > 0) {
      const lastMessage = result.history[result.history.length - 1];
      if (lastMessage?.content) {
        responseText = lastMessage.content;
        jsonData = tryExtractJson(responseText);
      }
    }
    // If JSON is invalid, retry with correction request
    let retries = 0;
    while (!jsonData && retries < maxRetries) {
      retries++;
      console.log(`      [${queryId}] Invalid JSON, requesting correction (retry ${retries}/${maxRetries})...`);
      result = await orchestrator.continueTask(
        `Your response was not valid JSON. Please output ONLY the JSON object with no markdown, no explanation, no thinking tags. Just the raw JSON starting with { and ending with }. Format:
 {"invoice_number":"X","invoice_date":"YYYY-MM-DD","vendor_name":"X","currency":"EUR","net_amount":0,"vat_amount":0,"total_amount":0}`
      );
      elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
      console.log(`      [${queryId}] Retry ${retries}: ${result.status} (${elapsed}s)`);
      responseText = result.result || '';
      if (responseText) {
        jsonData = tryExtractJson(responseText);
      }
      if (!jsonData && result.history?.length > 0) {
        const lastMessage = result.history[result.history.length - 1];
        if (lastMessage?.content) {
          responseText = lastMessage.content;
          jsonData = tryExtractJson(responseText);
        }
      }
    }
    if (!jsonData) {
      console.log(`      [${queryId}] Failed to get valid JSON after ${retries} retries`);
      return null;
    }
    console.log(`      [${queryId}] Valid JSON extracted`);
    return {
      invoice_number: extractInvoiceNumber(String(jsonData.invoice_number || '')),
      invoice_date: extractDate(String(jsonData.invoice_date || '')),
      vendor_name: String(jsonData.vendor_name || '').replace(/\*\*/g, '').replace(/`/g, '').trim(),
      currency: extractCurrency(String(jsonData.currency || '')),
      net_amount: parseAmount(jsonData.net_amount as string | number),
      vat_amount: parseAmount(jsonData.vat_amount as string | number),
      total_amount: parseAmount(jsonData.total_amount as string | number),
    };
  } catch (error) {
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    console.log(`      [${queryId}] ERROR: ${error} (${elapsed}s)`);
    throw error;
  }
 }
 /**
 * Extract invoice (single pass - GPT-OSS is more reliable)
 */
 async function extractInvoice(markdown: string, docName: string): Promise<IInvoice> {
  console.log(`    [${docName}] Extracting...`);
  const invoice = await extractInvoiceFromMarkdown(markdown, docName);
  if (!invoice) {
    return {
      invoice_number: '',
      invoice_date: '',
      vendor_name: '',
      currency: 'EUR',
      net_amount: 0,
      vat_amount: 0,
      total_amount: 0,
    };
  }
  console.log(`    [${docName}] Extracted: ${invoice.invoice_number}`);
  return invoice;
 }
 /**
 * Normalize date to YYYY-MM-DD
 */
 function normalizeDate(dateStr: string | null): string {
  if (!dateStr) return '';
  if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
  const monthMap: Record<string, string> = {
    JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
    JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
  };
  let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
  if (match) {
    return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
  }
  match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
  if (match) {
    return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
  }
  return dateStr;
 }
 /**
 * Compare extracted invoice against expected
 */
 function compareInvoice(
  extracted: IInvoice,
  expected: IInvoice
 ): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  if (extNum !== expNum) {
    errors.push(`invoice_number: exp "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
    errors.push(`invoice_date: exp "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: exp ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: exp "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find all test cases
 */
 function findTestCases(): ITestCase[] {
  const testDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(testDir)) return [];
  const files = fs.readdirSync(testDir);
  const testCases: ITestCase[] = [];
  for (const pdf of files.filter((f) => f.endsWith('.pdf'))) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases.sort((a, b) => a.name.localeCompare(b.name));
 }
 // ============ TESTS ============
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} invoice test cases\n`);
 // Ensure cache directory exists
 if (!fs.existsSync(MD_CACHE_DIR)) {
  fs.mkdirSync(MD_CACHE_DIR, { recursive: true });
 }
 // -------- STAGE 1: OCR with Nanonets --------
 tap.test('Stage 1: Convert invoices to markdown (with caching)', async () => {
  console.log('\n========== STAGE 1: Nanonets OCR ==========\n');
  // Check which invoices need OCR conversion
  const needsConversion: ITestCase[] = [];
  let cachedCount = 0;
  for (const tc of testCases) {
    const mdPath = path.join(MD_CACHE_DIR, `${tc.name}.md`);
    if (fs.existsSync(mdPath)) {
      cachedCount++;
      tc.markdownPath = mdPath;
      console.log(`  [CACHED] ${tc.name} - using cached markdown`);
    } else {
      needsConversion.push(tc);
    }
  }
  console.log(`\n  Summary: ${cachedCount} cached, ${needsConversion.length} need conversion\n`);
  if (needsConversion.length === 0) {
    console.log('  All invoices already cached, skipping Nanonets OCR\n');
    return;
  }
  // Start Nanonets only if there are files to convert
  console.log('  Starting Nanonets for OCR conversion...\n');
  const ok = await ensureNanonetsOcr();
  expect(ok).toBeTrue();
  // Convert only the invoices that need conversion
  for (const tc of needsConversion) {
    console.log(`\n    === ${tc.name} ===`);
    const images = convertPdfToImages(tc.pdfPath);
    console.log(`      Pages: ${images.length}`);
    const markdown = await convertDocumentToMarkdown(images, tc.name);
    const mdPath = path.join(MD_CACHE_DIR, `${tc.name}.md`);
    fs.writeFileSync(mdPath, markdown);
    tc.markdownPath = mdPath;
    console.log(`      Saved: ${mdPath}`);
  }
  console.log(`\n  Stage 1 complete: ${needsConversion.length} invoices converted to markdown\n`);
 });
 tap.test('Stage 1: Stop Nanonets', async () => {
  stopNanonets();
  await new Promise(resolve => setTimeout(resolve, 3000));
  expect(isContainerRunning('nanonets-test')).toBeFalse();
 });
 // -------- STAGE 2: Extraction with GPT-OSS 20B --------
 tap.test('Stage 2: Setup Ollama + GPT-OSS 20B', async () => {
  console.log('\n========== STAGE 2: GPT-OSS 20B Extraction ==========\n');
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  const extractionOk = await ensureExtractionModel();
  expect(extractionOk).toBeTrue();
  // Initialize SmartAi and DualAgentOrchestrator
  console.log('  [SmartAgent] Starting SmartAi...');
  await smartAi.start();
  console.log('  [SmartAgent] Creating DualAgentOrchestrator with native tool calling...');
  orchestrator = new DualAgentOrchestrator({
    smartAiInstance: smartAi,
    defaultProvider: 'ollama',
    guardianPolicyPrompt: `
      JSON EXTRACTION POLICY:
      - APPROVE all JSON extraction tasks
      - APPROVE all json.validate tool calls
      - This is a read-only operation - no file system or network access needed
      - The task is to extract structured data from document text
    `,
    driverSystemMessage: `You are a precise JSON extraction assistant. Your only job is to extract invoice data from documents.
 CRITICAL RULES:
 1. Output valid JSON with the exact format requested
 2. If you cannot find a value, use empty string "" or 0 for numbers
 3. Before completing, validate your JSON using the json_validate tool
 4. Only complete after validation passes`,
    maxIterations: 5,
    // Enable native tool calling for GPT-OSS (uses Harmony format instead of XML)
    useNativeToolCalling: true,
    // Enable streaming for real-time progress visibility
    onToken: (token, source) => {
      if (source === 'driver') {
        process.stdout.write(token);
      }
    },
    // Enable progress events to see tool calls
    onProgress: (event: { logMessage: string }) => {
      console.log(event.logMessage);
    },
  });
  // Register JsonValidatorTool for self-validation
  orchestrator.registerTool(new JsonValidatorTool());
  console.log('  [SmartAgent] Starting orchestrator...');
  await orchestrator.start();
  console.log('  [SmartAgent] Ready for extraction');
 });
 let passedCount = 0;
 let failedCount = 0;
 const processingTimes: number[] = [];
 for (const tc of testCases) {
  tap.test(`Stage 2: Extract ${tc.name}`, async () => {
    const expected: IInvoice = JSON.parse(fs.readFileSync(tc.jsonPath, 'utf-8'));
    console.log(`\n    === ${tc.name} ===`);
    console.log(`    Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
    const startTime = Date.now();
    const mdPath = path.join(MD_CACHE_DIR, `${tc.name}.md`);
    if (!fs.existsSync(mdPath)) {
      throw new Error(`Markdown not found: ${mdPath}. Run Stage 1 first.`);
    }
    const markdown = fs.readFileSync(mdPath, 'utf-8');
    console.log(`    Markdown: ${markdown.length} chars`);
    const extracted = await extractInvoice(markdown, tc.name);
    const elapsedMs = Date.now() - startTime;
    processingTimes.push(elapsedMs);
    console.log(`    Extracted: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`    Result: MATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`    Result: MISMATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
      result.errors.forEach(e => console.log(`      - ${e}`));
    }
    expect(result.match).toBeTrue();
  });
 }
 tap.test('Summary', async () => {
  // Cleanup orchestrator and SmartAi
  if (orchestrator) {
    console.log('\n  [SmartAgent] Stopping orchestrator...');
    await orchestrator.stop();
  }
  console.log('  [SmartAgent] Stopping SmartAi...');
  await smartAi.stop();
  const totalInvoices = testCases.length;
  const accuracy = totalInvoices > 0 ? (passedCount / totalInvoices) * 100 : 0;
  const totalTimeMs = processingTimes.reduce((a, b) => a + b, 0);
  const avgTimeSec = processingTimes.length > 0 ? totalTimeMs / processingTimes.length / 1000 : 0;
  console.log(`\n========================================`);
  console.log(`   Invoice Summary (Nanonets + GPT-OSS 20B)`);
  console.log(`========================================`);
  console.log(`  Stage 1:   Nanonets-OCR-s (doc -> md)`);
  console.log(`  Stage 2:   GPT-OSS 20B + SmartAgent (md -> JSON)`);
  console.log(`  Passed:    ${passedCount}/${totalInvoices}`);
  console.log(`  Failed:    ${failedCount}/${totalInvoices}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`----------------------------------------`);
  console.log(`  Total time:   ${(totalTimeMs / 1000).toFixed(1)}s`);
  console.log(`  Avg per inv:  ${avgTimeSec.toFixed(1)}s`);
  console.log(`========================================\n`);
  console.log(`  Cache location: ${MD_CACHE_DIR}\n`);
 });
 export default tap.start();
--- a/test/test.invoices.qwen3vl.ts
+++ b/test/test.invoices.qwen3vl.ts
@@ -0,0 +1,351 @@
 /**
 * Invoice extraction using Qwen3-VL 8B Vision (Direct)
 *
 * Multi-query approach: 5 parallel simple queries to avoid token exhaustion.
 * Single pass, no consensus voting.
 */
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 import { ensureMiniCpm } from './helpers/docker.js';
 const OLLAMA_URL = 'http://localhost:11434';
 const VISION_MODEL = 'qwen3-vl:8b';
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 /**
 * Convert PDF to PNG images using ImageMagick
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    // 150 DPI is sufficient for invoice extraction, reduces context size
    execSync(
      `convert -density 150 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Query Qwen3-VL for a single field
 * Uses simple prompts to minimize thinking tokens
 */
 async function queryField(images: string[], question: string): Promise<string> {
  const response = await fetch(`${OLLAMA_URL}/api/chat`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: VISION_MODEL,
      messages: [{
        role: 'user',
        content: `${question} Reply with just the value, nothing else.`,
        images: images,
      }],
      stream: false,
      options: {
        num_predict: 500,
        temperature: 0.1,
      },
    }),
  });
  if (!response.ok) {
    throw new Error(`Ollama API error: ${response.status}`);
  }
  const data = await response.json();
  return (data.message?.content || '').trim();
 }
 /**
 * Extract invoice data using multiple simple queries
 * Each query asks for 1-2 fields to minimize thinking tokens
 * (Qwen3's thinking mode uses all tokens on complex prompts)
 */
 async function extractInvoiceFromImages(images: string[]): Promise<IInvoice> {
  console.log(`    [Vision] Processing ${images.length} page(s) with Qwen3-VL (multi-query)`);
  // Query each field separately to avoid excessive thinking tokens
  // Use explicit questions to avoid confusion between similar fields
  // Log each result as it comes in (not waiting for all to complete)
  const queryAndLog = async (name: string, question: string): Promise<string> => {
    const result = await queryField(images, question);
    console.log(`    [Query] ${name}: "${result}"`);
    return result;
  };
  const [invoiceNum, invoiceDate, vendor, currency, totalAmount, netAmount, vatAmount] = await Promise.all([
    queryAndLog('Invoice Number', 'What is the INVOICE NUMBER (not VAT number, not customer ID)? Look for "Invoice No", "Invoice #", "Rechnung Nr", "Facture". Just the number/code.'),
    queryAndLog('Invoice Date  ', 'What is the INVOICE DATE (not due date, not delivery date)? The date the invoice was issued. Format: YYYY-MM-DD'),
    queryAndLog('Vendor        ', 'What company ISSUED this invoice (the seller/vendor, not the buyer)? Look at the letterhead or "From" section.'),
    queryAndLog('Currency      ', 'What CURRENCY is used? Look for € (EUR), $ (USD), or £ (GBP). Answer with 3-letter code: EUR, USD, or GBP'),
    queryAndLog('Total Amount  ', 'What is the TOTAL AMOUNT INCLUDING TAX (the final amount to pay, with VAT/tax included)? Just the number, e.g. 24.99'),
    queryAndLog('Net Amount    ', 'What is the NET AMOUNT (subtotal before VAT/tax)? Just the number, e.g. 20.99'),
    queryAndLog('VAT Amount    ', 'What is the VAT/TAX AMOUNT? Just the number, e.g. 4.00'),
  ]);
  // Parse amount from string (handles European format)
  const parseAmount = (s: string): number => {
    if (!s) return 0;
    // Extract number from the response
    const match = s.match(/([\d.,]+)/);
    if (!match) return 0;
    const numStr = match[1];
    // Handle European format: 1.234,56 → 1234.56
    const normalized = numStr.includes(',') && numStr.indexOf(',') > numStr.lastIndexOf('.')
      ? numStr.replace(/\./g, '').replace(',', '.')
      : numStr.replace(/,/g, '');
    return parseFloat(normalized) || 0;
  };
  // Extract invoice number from potentially verbose response
  const extractInvoiceNumber = (s: string): string => {
    let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
    // Look for common invoice number patterns
    const patterns = [
      /\b([A-Z]{2,3}\d{10,})\b/i,           // IEE2022006460244
      /\b([A-Z]\d{8,})\b/i,                  // R0014359508
      /\b(INV[-\s]?\d{4}[-\s]?\d+)\b/i,     // INV-2024-001
      /\b(\d{7,})\b/,                        // 1579087430
    ];
    for (const pattern of patterns) {
      const match = clean.match(pattern);
      if (match) return match[1];
    }
    return clean.replace(/[^A-Z0-9-]/gi, '').trim() || clean;
  };
  // Extract date (YYYY-MM-DD) from response
  const extractDate = (s: string): string => {
    let clean = s.replace(/\*\*/g, '').replace(/`/g, '').trim();
    const isoMatch = clean.match(/(\d{4}-\d{2}-\d{2})/);
    if (isoMatch) return isoMatch[1];
    return clean.replace(/[^\d-]/g, '').trim();
  };
  // Extract currency
  const extractCurrency = (s: string): string => {
    const upper = s.toUpperCase();
    if (upper.includes('EUR') || upper.includes('€')) return 'EUR';
    if (upper.includes('USD') || upper.includes('$')) return 'USD';
    if (upper.includes('GBP') || upper.includes('£')) return 'GBP';
    return 'EUR';
  };
  return {
    invoice_number: extractInvoiceNumber(invoiceNum),
    invoice_date: extractDate(invoiceDate),
    vendor_name: vendor.replace(/\*\*/g, '').replace(/`/g, '').trim() || '',
    currency: extractCurrency(currency),
    net_amount: parseAmount(netAmount),
    vat_amount: parseAmount(vatAmount),
    total_amount: parseAmount(totalAmount),
  };
 }
 /**
 * Normalize date to YYYY-MM-DD
 */
 function normalizeDate(dateStr: string | null): string {
  if (!dateStr) return '';
  if (/^\d{4}-\d{2}-\d{2}$/.test(dateStr)) return dateStr;
  const monthMap: Record<string, string> = {
    JAN: '01', FEB: '02', MAR: '03', APR: '04', MAY: '05', JUN: '06',
    JUL: '07', AUG: '08', SEP: '09', OCT: '10', NOV: '11', DEC: '12',
  };
  let match = dateStr.match(/^(\d{1,2})-([A-Z]{3})-(\d{4})$/i);
  if (match) {
    return `${match[3]}-${monthMap[match[2].toUpperCase()] || '01'}-${match[1].padStart(2, '0')}`;
  }
  match = dateStr.match(/^(\d{1,2})[\/.](\d{1,2})[\/.](\d{4})$/);
  if (match) {
    return `${match[3]}-${match[2].padStart(2, '0')}-${match[1].padStart(2, '0')}`;
  }
  return dateStr;
 }
 /**
 * Compare extracted vs expected
 */
 function compareInvoice(extracted: IInvoice, expected: IInvoice): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  if (extNum !== expNum) {
    errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  if (normalizeDate(extracted.invoice_date) !== normalizeDate(expected.invoice_date)) {
    errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find test cases
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(testDir)) return [];
  const files = fs.readdirSync(testDir);
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of files.filter((f) => f.endsWith('.pdf'))) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases.sort((a, b) => a.name.localeCompare(b.name));
 }
 /**
 * Ensure Qwen3-VL 8B model is available
 */
 async function ensureQwen3Vl(): Promise<boolean> {
  try {
    const response = await fetch(`${OLLAMA_URL}/api/tags`);
    if (response.ok) {
      const data = await response.json();
      const models = data.models || [];
      if (models.some((m: { name: string }) => m.name === VISION_MODEL)) {
        console.log(`[Ollama] Model already available: ${VISION_MODEL}`);
        return true;
      }
    }
  } catch {
    console.log('[Ollama] Cannot check models');
    return false;
  }
  console.log(`[Ollama] Pulling model: ${VISION_MODEL}...`);
  const pullResponse = await fetch(`${OLLAMA_URL}/api/pull`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ name: VISION_MODEL, stream: false }),
  });
  return pullResponse.ok;
 }
 // Tests
 tap.test('setup: ensure Qwen3-VL is running', async () => {
  console.log('\n[Setup] Checking Qwen3-VL 8B...\n');
  // Ensure Ollama service is running
  const ollamaOk = await ensureMiniCpm();
  expect(ollamaOk).toBeTrue();
  // Ensure Qwen3-VL 8B model
  const visionOk = await ensureQwen3Vl();
  expect(visionOk).toBeTrue();
  console.log('\n[Setup] Ready!\n');
 });
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} invoice test cases (Qwen3-VL Vision)\n`);
 let passedCount = 0;
 let failedCount = 0;
 const times: number[] = [];
 for (const testCase of testCases) {
  tap.test(`should extract invoice: ${testCase.name}`, async () => {
    const expected: IInvoice = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
    const start = Date.now();
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`  Pages: ${images.length}`);
    const extracted = await extractInvoiceFromImages(images);
    console.log(`  Extracted: ${extracted.invoice_number} | ${extracted.invoice_date} | ${extracted.total_amount} ${extracted.currency}`);
    const elapsed = Date.now() - start;
    times.push(elapsed);
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`  Result: MATCH (${(elapsed / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`  Result: MISMATCH (${(elapsed / 1000).toFixed(1)}s)`);
      result.errors.forEach((e) => console.log(`    - ${e}`));
    }
    expect(result.match).toBeTrue();
  });
 }
 tap.test('summary', async () => {
  const total = testCases.length;
  const accuracy = total > 0 ? (passedCount / total) * 100 : 0;
  const totalTime = times.reduce((a, b) => a + b, 0) / 1000;
  const avgTime = times.length > 0 ? totalTime / times.length : 0;
  console.log(`\n======================================================`);
  console.log(`   Invoice Extraction Summary (Qwen3-VL Vision)`);
  console.log(`======================================================`);
  console.log(`  Method:    Multi-query (single pass)`);
  console.log(`  Passed:    ${passedCount}/${total}`);
  console.log(`  Failed:    ${failedCount}/${total}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`------------------------------------------------------`);
  console.log(`  Total time:   ${totalTime.toFixed(1)}s`);
  console.log(`  Avg per inv:  ${avgTime.toFixed(1)}s`);
  console.log(`======================================================\n`);
 });
 export default tap.start();
--- a/test/test.invoices.ts
+++ b/test/test.invoices.ts
@@ -1,431 +0,0 @@
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 const OLLAMA_URL = 'http://localhost:11434';
 const MODEL = 'openbmb/minicpm-v4.5:q8_0';
 const PADDLEOCR_URL = 'http://localhost:5000';
 interface IInvoice {
  invoice_number: string;
  invoice_date: string;
  vendor_name: string;
  currency: string;
  net_amount: number;
  vat_amount: number;
  total_amount: number;
 }
 /**
 * Extract OCR text from an image using PaddleOCR
 */
 async function extractOcrText(imageBase64: string): Promise<string> {
  try {
    const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ image: imageBase64 }),
    });
    if (!response.ok) return '';
    const data = await response.json();
    if (data.success && data.results) {
      return data.results.map((r: { text: string }) => r.text).join('\n');
    }
  } catch {
    // PaddleOCR unavailable
  }
  return '';
 }
 /**
 * Build prompt with optional OCR text
 */
 function buildPrompt(ocrText: string): string {
  const base = `/nothink
 You are an invoice parser. Extract the following fields from this invoice:
 1. invoice_number: The invoice/receipt number
 2. invoice_date: Date in YYYY-MM-DD format
 3. vendor_name: Company that issued the invoice
 4. currency: EUR, USD, etc.
 5. net_amount: Amount before tax (if shown)
 6. vat_amount: Tax/VAT amount (if shown, 0 if reverse charge or no tax)
 7. total_amount: Final amount due
 Return ONLY valid JSON in this exact format:
 {"invoice_number":"XXX","invoice_date":"YYYY-MM-DD","vendor_name":"Company Name","currency":"EUR","net_amount":100.00,"vat_amount":19.00,"total_amount":119.00}
 If a field is not visible, use null for strings or 0 for numbers.
 No explanation, just the JSON object.`;
  if (ocrText) {
    // Limit OCR text to prevent context overflow
    const maxOcrLength = 4000;
    const truncatedOcr = ocrText.length > maxOcrLength
      ? ocrText.substring(0, maxOcrLength) + '\n... (truncated)'
      : ocrText;
    return `${base}
 OCR text extracted from the invoice (use for reference):
 ---
 ${truncatedOcr}
 ---
 Cross-reference the image with the OCR text above for accuracy.`;
  }
  return base;
 }
 /**
 * Convert PDF to PNG images using ImageMagick
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    execSync(
      `convert -density 200 -quality 90 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Single extraction pass
 */
 async function extractOnce(images: string[], passNum: number, ocrText: string = ''): Promise<IInvoice> {
  const payload = {
    model: MODEL,
    prompt: buildPrompt(ocrText),
    images,
    stream: true,
    options: {
      num_predict: 2048,
      temperature: 0.1,
    },
  };
  const response = await fetch(`${OLLAMA_URL}/api/generate`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(payload),
  });
  if (!response.ok) {
    throw new Error(`Ollama API error: ${response.status}`);
  }
  const reader = response.body?.getReader();
  if (!reader) {
    throw new Error('No response body');
  }
  const decoder = new TextDecoder();
  let fullText = '';
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    const chunk = decoder.decode(value, { stream: true });
    const lines = chunk.split('\n').filter((l) => l.trim());
    for (const line of lines) {
      try {
        const json = JSON.parse(line);
        if (json.response) {
          fullText += json.response;
        }
      } catch {
        // Skip invalid JSON lines
      }
    }
  }
  // Extract JSON from response
  const startIdx = fullText.indexOf('{');
  const endIdx = fullText.lastIndexOf('}') + 1;
  if (startIdx < 0 || endIdx <= startIdx) {
    throw new Error(`No JSON object found in response: ${fullText.substring(0, 200)}`);
  }
  const jsonStr = fullText.substring(startIdx, endIdx);
  return JSON.parse(jsonStr);
 }
 /**
 * Create a hash of invoice for comparison (using key fields)
 */
 function hashInvoice(invoice: IInvoice): string {
  return `${invoice.invoice_number}|${invoice.invoice_date}|${invoice.total_amount.toFixed(2)}`;
 }
 /**
 * Extract with majority voting - run until 2 passes match
 * Optimization: Run Pass 1, OCR, and Pass 2 (after OCR) in parallel
 */
 async function extractWithConsensus(images: string[], invoiceName: string, maxPasses: number = 5): Promise<IInvoice> {
  const results: Array<{ invoice: IInvoice; hash: string }> = [];
  const hashCounts: Map<string, number> = new Map();
  const addResult = (invoice: IInvoice, passLabel: string): number => {
    const hash = hashInvoice(invoice);
    results.push({ invoice, hash });
    hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);
    console.log(`  [${passLabel}] ${invoice.invoice_number} | ${invoice.invoice_date} | ${invoice.total_amount} ${invoice.currency}`);
    return hashCounts.get(hash)!;
  };
  // OPTIMIZATION: Run Pass 1 (no OCR) in parallel with OCR -> Pass 2 (with OCR)
  let ocrText = '';
  const pass1Promise = extractOnce(images, 1, '').catch((err) => ({ error: err }));
  // OCR then immediately Pass 2
  const ocrThenPass2Promise = (async () => {
    ocrText = await extractOcrText(images[0]);
    if (ocrText) {
      console.log(`  [OCR] Extracted ${ocrText.split('\n').length} text lines`);
    }
    return extractOnce(images, 2, ocrText).catch((err) => ({ error: err }));
  })();
  // Wait for both to complete
  const [pass1Result, pass2Result] = await Promise.all([pass1Promise, ocrThenPass2Promise]);
  // Process Pass 1 result
  if ('error' in pass1Result) {
    console.log(`  [Pass 1] Error: ${(pass1Result as {error: unknown}).error}`);
  } else {
    const count = addResult(pass1Result as IInvoice, 'Pass 1');
    if (count >= 2) {
      console.log(`  [Consensus] Reached after parallel passes`);
      return pass1Result as IInvoice;
    }
  }
  // Process Pass 2 result
  if ('error' in pass2Result) {
    console.log(`  [Pass 2+OCR] Error: ${(pass2Result as {error: unknown}).error}`);
  } else {
    const count = addResult(pass2Result as IInvoice, 'Pass 2+OCR');
    if (count >= 2) {
      console.log(`  [Consensus] Reached after parallel passes`);
      return pass2Result as IInvoice;
    }
  }
  // Continue with passes 3+ using OCR text if no consensus yet
  for (let pass = 3; pass <= maxPasses; pass++) {
    try {
      const invoice = await extractOnce(images, pass, ocrText);
      const count = addResult(invoice, `Pass ${pass}+OCR`);
      if (count >= 2) {
        console.log(`  [Consensus] Reached after ${pass} passes`);
        return invoice;
      }
    } catch (err) {
      console.log(`  [Pass ${pass}] Error: ${err}`);
    }
  }
  // No consensus reached - return the most common result
  let bestHash = '';
  let bestCount = 0;
  for (const [hash, count] of hashCounts) {
    if (count > bestCount) {
      bestCount = count;
      bestHash = hash;
    }
  }
  if (!bestHash) {
    throw new Error(`No valid results for ${invoiceName}`);
  }
  const best = results.find((r) => r.hash === bestHash)!;
  console.log(`  [No consensus] Using most common result (${bestCount}/${maxPasses} passes)`);
  return best.invoice;
 }
 /**
 * Compare extracted invoice against expected
 */
 function compareInvoice(
  extracted: IInvoice,
  expected: IInvoice
 ): { match: boolean; errors: string[] } {
  const errors: string[] = [];
  // Compare invoice number (normalize by removing spaces and case)
  const extNum = extracted.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  const expNum = expected.invoice_number?.replace(/\s/g, '').toLowerCase() || '';
  if (extNum !== expNum) {
    errors.push(`invoice_number: expected "${expected.invoice_number}", got "${extracted.invoice_number}"`);
  }
  // Compare date
  if (extracted.invoice_date !== expected.invoice_date) {
    errors.push(`invoice_date: expected "${expected.invoice_date}", got "${extracted.invoice_date}"`);
  }
  // Compare total amount (with tolerance)
  if (Math.abs(extracted.total_amount - expected.total_amount) > 0.02) {
    errors.push(`total_amount: expected ${expected.total_amount}, got ${extracted.total_amount}`);
  }
  // Compare currency
  if (extracted.currency?.toUpperCase() !== expected.currency?.toUpperCase()) {
    errors.push(`currency: expected "${expected.currency}", got "${extracted.currency}"`);
  }
  return { match: errors.length === 0, errors };
 }
 /**
 * Find all test cases (PDF + JSON pairs) in .nogit/invoices/
 * Priority invoices (like vodafone) run first for quick feedback
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit/invoices');
  if (!fs.existsSync(testDir)) {
    return [];
  }
  const files = fs.readdirSync(testDir);
  const pdfFiles = files.filter((f) => f.endsWith('.pdf'));
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of pdfFiles) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  // Sort with priority invoices first, then alphabetically
  const priorityPrefixes = ['vodafone'];
  testCases.sort((a, b) => {
    const aPriority = priorityPrefixes.findIndex((p) => a.name.startsWith(p));
    const bPriority = priorityPrefixes.findIndex((p) => b.name.startsWith(p));
    // Both have priority - sort by priority order
    if (aPriority >= 0 && bPriority >= 0) return aPriority - bPriority;
    // Only a has priority - a comes first
    if (aPriority >= 0) return -1;
    // Only b has priority - b comes first
    if (bPriority >= 0) return 1;
    // Neither has priority - alphabetical
    return a.name.localeCompare(b.name);
  });
  return testCases;
 }
 // Tests
 tap.test('should connect to Ollama API', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  expect(response.ok).toBeTrue();
  const data = await response.json();
  expect(data.models).toBeArray();
 });
 tap.test('should have MiniCPM-V 4.5 model loaded', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  const data = await response.json();
  const modelNames = data.models.map((m: { name: string }) => m.name);
  expect(modelNames.some((name: string) => name.includes('minicpm-v4.5'))).toBeTrue();
 });
 // Dynamic test for each PDF/JSON pair
 const testCases = findTestCases();
 console.log(`\nFound ${testCases.length} invoice test cases\n`);
 let passedCount = 0;
 let failedCount = 0;
 const processingTimes: number[] = [];
 for (const testCase of testCases) {
  tap.test(`should extract invoice: ${testCase.name}`, async () => {
    // Load expected data
    const expected: IInvoice = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.invoice_number} | ${expected.invoice_date} | ${expected.total_amount} ${expected.currency}`);
    const startTime = Date.now();
    // Convert PDF to images
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`  Pages: ${images.length}`);
    // Extract with consensus voting
    const extracted = await extractWithConsensus(images, testCase.name);
    const endTime = Date.now();
    const elapsedMs = endTime - startTime;
    processingTimes.push(elapsedMs);
    // Compare results
    const result = compareInvoice(extracted, expected);
    if (result.match) {
      passedCount++;
      console.log(`  Result: MATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
    } else {
      failedCount++;
      console.log(`  Result: MISMATCH (${(elapsedMs / 1000).toFixed(1)}s)`);
      result.errors.forEach((e) => console.log(`    - ${e}`));
    }
    // Assert match
    expect(result.match).toBeTrue();
  });
 }
 tap.test('summary', async () => {
  const totalInvoices = testCases.length;
  const accuracy = totalInvoices > 0 ? (passedCount / totalInvoices) * 100 : 0;
  const totalTimeMs = processingTimes.reduce((a, b) => a + b, 0);
  const avgTimeMs = processingTimes.length > 0 ? totalTimeMs / processingTimes.length : 0;
  const avgTimeSec = avgTimeMs / 1000;
  const totalTimeSec = totalTimeMs / 1000;
  console.log(`\n========================================`);
  console.log(`        Invoice Extraction Summary`);
  console.log(`========================================`);
  console.log(`  Passed:    ${passedCount}/${totalInvoices}`);
  console.log(`  Failed:    ${failedCount}/${totalInvoices}`);
  console.log(`  Accuracy:  ${accuracy.toFixed(1)}%`);
  console.log(`----------------------------------------`);
  console.log(`  Total time:   ${totalTimeSec.toFixed(1)}s`);
  console.log(`  Avg per inv:  ${avgTimeSec.toFixed(1)}s`);
  console.log(`========================================\n`);
 });
 export default tap.start();
--- a/test/test.node.ts
+++ b/test/test.node.ts
@@ -1,439 +0,0 @@
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 const OLLAMA_URL = 'http://localhost:11434';
 const MODEL = 'openbmb/minicpm-v4.5:q8_0';
 const PADDLEOCR_URL = 'http://localhost:5000';
 // Prompt for visual extraction (with images)
 const VISUAL_EXTRACT_PROMPT = `/nothink
 You are a bank statement parser. Extract EVERY transaction from the table.
 Read the Amount column carefully:
 - "- 21,47 €" means DEBIT, output as: -21.47
 - "+ 1.000,00 €" means CREDIT, output as: 1000.00
 - European format: comma = decimal point
 For each row output: {"date":"YYYY-MM-DD","counterparty":"NAME","amount":-21.47}
 Do not skip any rows. Return ONLY the JSON array, no explanation.`;
 // Prompt for OCR-only extraction (no images)
 const OCR_EXTRACT_PROMPT = `/nothink
 You are a bank statement parser. Extract EVERY transaction from the OCR text below.
 Read the Amount values carefully:
 - "- 21,47 €" means DEBIT, output as: -21.47
 - "+ 1.000,00 €" means CREDIT, output as: 1000.00
 - European format: comma = decimal point
 For each transaction output: {"date":"YYYY-MM-DD","counterparty":"NAME","amount":-21.47}
 Do not skip any transactions. Return ONLY the JSON array, no explanation.`;
 /**
 * Build prompt for OCR-only extraction (no images)
 */
 function buildOcrOnlyPrompt(ocrText: string): string {
  // Limit OCR text to prevent context overflow
  const maxOcrLength = 12000;
  const truncatedOcr = ocrText.length > maxOcrLength
    ? ocrText.substring(0, maxOcrLength) + '\n... (truncated)'
    : ocrText;
  return `${OCR_EXTRACT_PROMPT}
 OCR text from bank statement:
 ---
 ${truncatedOcr}
 ---`;
 }
 /**
 * Extract OCR text from an image using PaddleOCR
 */
 async function extractOcrText(imageBase64: string): Promise<string> {
  try {
    const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ image: imageBase64 }),
    });
    if (!response.ok) return '';
    const data = await response.json();
    if (data.success && data.results) {
      return data.results.map((r: { text: string }) => r.text).join('\n');
    }
  } catch {
    // PaddleOCR unavailable
  }
  return '';
 }
 interface ITransaction {
  date: string;
  counterparty: string;
  amount: number;
 }
 /**
 * Convert PDF to PNG images using ImageMagick
 */
 function convertPdfToImages(pdfPath: string): string[] {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPattern = path.join(tempDir, 'page-%d.png');
  try {
    execSync(
      `convert -density 300 -quality 100 "${pdfPath}" -background white -alpha remove "${outputPattern}"`,
      { stdio: 'pipe' }
    );
    const files = fs.readdirSync(tempDir).filter((f) => f.endsWith('.png')).sort();
    const images: string[] = [];
    for (const file of files) {
      const imagePath = path.join(tempDir, file);
      const imageData = fs.readFileSync(imagePath);
      images.push(imageData.toString('base64'));
    }
    return images;
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Visual extraction pass (with images)
 */
 async function extractVisual(images: string[], passLabel: string): Promise<ITransaction[]> {
  const payload = {
    model: MODEL,
    prompt: VISUAL_EXTRACT_PROMPT,
    images,
    stream: true,
    options: {
      num_predict: 16384,
      temperature: 0.1,
    },
  };
  return doExtraction(payload, passLabel);
 }
 /**
 * OCR-only extraction pass (no images, just text)
 */
 async function extractFromOcr(ocrText: string, passLabel: string): Promise<ITransaction[]> {
  const payload = {
    model: MODEL,
    prompt: buildOcrOnlyPrompt(ocrText),
    stream: true,
    options: {
      num_predict: 16384,
      temperature: 0.1,
    },
  };
  return doExtraction(payload, passLabel);
 }
 /**
 * Common extraction logic
 */
 async function doExtraction(payload: object, passLabel: string): Promise<ITransaction[]> {
  const response = await fetch(`${OLLAMA_URL}/api/generate`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(payload),
  });
  if (!response.ok) {
    throw new Error(`Ollama API error: ${response.status}`);
  }
  const reader = response.body?.getReader();
  if (!reader) {
    throw new Error('No response body');
  }
  const decoder = new TextDecoder();
  let fullText = '';
  let lineBuffer = '';
  console.log(`[${passLabel}] Extracting...`);
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    const chunk = decoder.decode(value, { stream: true });
    const lines = chunk.split('\n').filter((l) => l.trim());
    for (const line of lines) {
      try {
        const json = JSON.parse(line);
        if (json.response) {
          fullText += json.response;
          lineBuffer += json.response;
          // Print complete lines
          if (lineBuffer.includes('\n')) {
            const parts = lineBuffer.split('\n');
            for (let i = 0; i < parts.length - 1; i++) {
              console.log(parts[i]);
            }
            lineBuffer = parts[parts.length - 1];
          }
        }
      } catch {
        // Skip invalid JSON lines
      }
    }
  }
  if (lineBuffer) {
    console.log(lineBuffer);
  }
  console.log('');
  const startIdx = fullText.indexOf('[');
  const endIdx = fullText.lastIndexOf(']') + 1;
  if (startIdx < 0 || endIdx <= startIdx) {
    throw new Error('No JSON array found in response');
  }
  return JSON.parse(fullText.substring(startIdx, endIdx));
 }
 /**
 * Create a hash of transactions for comparison
 */
 function hashTransactions(transactions: ITransaction[]): string {
  return transactions
    .map((t) => `${t.date}|${t.amount.toFixed(2)}`)
    .sort()
    .join(';');
 }
 /**
 * Extract with majority voting - run until 2 passes match
 * Strategy: Pass 1 = Visual (images), Pass 2 = OCR-only (text), Pass 3+ = Visual
 */
 async function extractWithConsensus(images: string[], maxPasses: number = 5): Promise<ITransaction[]> {
  const results: Array<{ transactions: ITransaction[]; hash: string }> = [];
  const hashCounts: Map<string, number> = new Map();
  const addResult = (transactions: ITransaction[], passLabel: string): number => {
    const hash = hashTransactions(transactions);
    results.push({ transactions, hash });
    hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);
    console.log(`[${passLabel}] Got ${transactions.length} transactions (hash: ${hash.substring(0, 20)}...)`);
    return hashCounts.get(hash)!;
  };
  // Run Pass 1 (Visual) in parallel with OCR extraction
  let ocrText = '';
  const pass1Promise = extractVisual(images, 'Pass 1 Visual').catch((err) => ({ error: err }));
  // Extract OCR from all pages
  const ocrPromise = (async () => {
    const ocrTexts: string[] = [];
    for (let i = 0; i < images.length; i++) {
      const pageOcr = await extractOcrText(images[i]);
      if (pageOcr) {
        ocrTexts.push(`--- Page ${i + 1} ---\n${pageOcr}`);
      }
    }
    ocrText = ocrTexts.join('\n\n');
    if (ocrText) {
      console.log(`[OCR] Extracted text from ${ocrTexts.length} page(s)`);
    }
    return ocrText;
  })();
  // Wait for Pass 1 and OCR to complete
  const [pass1Result] = await Promise.all([pass1Promise, ocrPromise]);
  // Process Pass 1 result
  if ('error' in pass1Result) {
    console.log(`[Pass 1] Error: ${(pass1Result as { error: unknown }).error}`);
  } else {
    addResult(pass1Result as ITransaction[], 'Pass 1 Visual');
  }
  // Pass 2: OCR-only (no images) - faster, different approach
  if (ocrText) {
    try {
      const pass2Result = await extractFromOcr(ocrText, 'Pass 2 OCR-only');
      const count = addResult(pass2Result, 'Pass 2 OCR-only');
      if (count >= 2) {
        console.log(`[Consensus] Visual and OCR extractions match!`);
        return pass2Result;
      }
    } catch (err) {
      console.log(`[Pass 2 OCR-only] Error: ${err}`);
    }
  }
  // Continue with visual passes 3+ if no consensus yet
  for (let pass = 3; pass <= maxPasses; pass++) {
    try {
      const transactions = await extractVisual(images, `Pass ${pass} Visual`);
      const count = addResult(transactions, `Pass ${pass} Visual`);
      if (count >= 2) {
        console.log(`[Consensus] Reached after ${pass} passes`);
        return transactions;
      }
      console.log(`[Pass ${pass}] No consensus yet, trying again...`);
    } catch (err) {
      console.log(`[Pass ${pass}] Error: ${err}`);
    }
  }
  // No consensus reached - return the most common result
  let bestHash = '';
  let bestCount = 0;
  for (const [hash, count] of hashCounts) {
    if (count > bestCount) {
      bestCount = count;
      bestHash = hash;
    }
  }
  if (!bestHash) {
    throw new Error('No valid results obtained');
  }
  const best = results.find((r) => r.hash === bestHash)!;
  console.log(`[No consensus] Using most common result (${bestCount}/${maxPasses} passes)`);
  return best.transactions;
 }
 /**
 * Compare extracted transactions against expected
 */
 function compareTransactions(
  extracted: ITransaction[],
  expected: ITransaction[]
 ): { matches: number; total: number; errors: string[] } {
  const errors: string[] = [];
  let matches = 0;
  for (let i = 0; i < expected.length; i++) {
    const exp = expected[i];
    const ext = extracted[i];
    if (!ext) {
      errors.push(`Missing transaction ${i}: ${exp.date} ${exp.counterparty}`);
      continue;
    }
    const dateMatch = ext.date === exp.date;
    const amountMatch = Math.abs(ext.amount - exp.amount) < 0.01;
    if (dateMatch && amountMatch) {
      matches++;
    } else {
      errors.push(
        `Mismatch at ${i}: expected ${exp.date}/${exp.amount}, got ${ext.date}/${ext.amount}`
      );
    }
  }
  if (extracted.length > expected.length) {
    errors.push(`Extra transactions: ${extracted.length - expected.length}`);
  }
  return { matches, total: expected.length, errors };
 }
 /**
 * Find all test cases (PDF + JSON pairs) in .nogit/
 */
 function findTestCases(): Array<{ name: string; pdfPath: string; jsonPath: string }> {
  const testDir = path.join(process.cwd(), '.nogit');
  if (!fs.existsSync(testDir)) {
    return [];
  }
  const files = fs.readdirSync(testDir);
  const pdfFiles = files.filter((f) => f.endsWith('.pdf'));
  const testCases: Array<{ name: string; pdfPath: string; jsonPath: string }> = [];
  for (const pdf of pdfFiles) {
    const baseName = pdf.replace('.pdf', '');
    const jsonFile = `${baseName}.json`;
    if (files.includes(jsonFile)) {
      testCases.push({
        name: baseName,
        pdfPath: path.join(testDir, pdf),
        jsonPath: path.join(testDir, jsonFile),
      });
    }
  }
  return testCases;
 }
 // Tests
 tap.test('should connect to Ollama API', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  expect(response.ok).toBeTrue();
  const data = await response.json();
  expect(data.models).toBeArray();
 });
 tap.test('should have MiniCPM-V 4.5 model loaded', async () => {
  const response = await fetch(`${OLLAMA_URL}/api/tags`);
  const data = await response.json();
  const modelNames = data.models.map((m: { name: string }) => m.name);
  expect(modelNames.some((name: string) => name.includes('minicpm-v4.5'))).toBeTrue();
 });
 // Dynamic test for each PDF/JSON pair
 const testCases = findTestCases();
 for (const testCase of testCases) {
  tap.test(`should extract transactions from ${testCase.name}`, async () => {
    // Load expected transactions
    const expected: ITransaction[] = JSON.parse(fs.readFileSync(testCase.jsonPath, 'utf-8'));
    console.log(`\n=== ${testCase.name} ===`);
    console.log(`Expected: ${expected.length} transactions`);
    // Convert PDF to images
    console.log('Converting PDF to images...');
    const images = convertPdfToImages(testCase.pdfPath);
    console.log(`Converted: ${images.length} pages\n`);
    // Extract with consensus voting
    const extracted = await extractWithConsensus(images);
    console.log(`\nFinal: ${extracted.length} transactions`);
    // Compare results
    const result = compareTransactions(extracted, expected);
    console.log(`Accuracy: ${result.matches}/${result.total}`);
    if (result.errors.length > 0) {
      console.log('Errors:');
      result.errors.forEach((e) => console.log(`  - ${e}`));
    }
    // Assert high accuracy
    const accuracy = result.matches / result.total;
    expect(accuracy).toBeGreaterThan(0.95);
    expect(extracted.length).toEqual(expected.length);
  });
 }
 export default tap.start();
--- a/test/test.paddleocr.ts
+++ b/test/test.paddleocr.ts
@@ -1,258 +0,0 @@
 import { tap, expect } from '@git.zone/tstest/tapbundle';
 import * as fs from 'fs';
 import * as path from 'path';
 import { execSync } from 'child_process';
 import * as os from 'os';
 const PADDLEOCR_URL = 'http://localhost:5000';
 interface IOCRResult {
  text: string;
  confidence: number;
  box: number[][];
 }
 interface IOCRResponse {
  success: boolean;
  results: IOCRResult[];
  error?: string;
 }
 interface IHealthResponse {
  status: string;
  model: string;
  language: string;
  gpu_enabled: boolean;
 }
 /**
 * Convert PDF first page to PNG using ImageMagick
 */
 function convertPdfToImage(pdfPath: string): string {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pdf-convert-'));
  const outputPath = path.join(tempDir, 'page.png');
  try {
    execSync(
      `convert -density 200 -quality 90 "${pdfPath}[0]" -background white -alpha remove "${outputPath}"`,
      { stdio: 'pipe' }
    );
    const imageData = fs.readFileSync(outputPath);
    return imageData.toString('base64');
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 /**
 * Create a simple test image with text using ImageMagick
 */
 function createTestImage(text: string): string {
  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-image-'));
  const outputPath = path.join(tempDir, 'test.png');
  try {
    execSync(
      `convert -size 400x100 xc:white -font DejaVu-Sans -pointsize 24 -fill black -gravity center -annotate 0 "${text}" "${outputPath}"`,
      { stdio: 'pipe' }
    );
    const imageData = fs.readFileSync(outputPath);
    return imageData.toString('base64');
  } finally {
    fs.rmSync(tempDir, { recursive: true, force: true });
  }
 }
 // Health check test
 tap.test('should respond to health check', async () => {
  const response = await fetch(`${PADDLEOCR_URL}/health`);
  expect(response.ok).toBeTrue();
  const data: IHealthResponse = await response.json();
  expect(data.status).toEqual('healthy');
  expect(data.model).toEqual('PP-OCRv4');
  expect(data.language).toBeTypeofString();
  expect(data.gpu_enabled).toBeTypeofBoolean();
  console.log(`PaddleOCR Status: ${data.status}`);
  console.log(`  Model: ${data.model}`);
  console.log(`  Language: ${data.language}`);
  console.log(`  GPU Enabled: ${data.gpu_enabled}`);
 });
 // Base64 OCR test
 tap.test('should perform OCR on base64 image', async () => {
  // Create a test image with known text
  const testText = 'Hello World 12345';
  console.log(`Creating test image with text: "${testText}"`);
  const imageBase64 = createTestImage(testText);
  const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ image: imageBase64 }),
  });
  expect(response.ok).toBeTrue();
  const data: IOCRResponse = await response.json();
  expect(data.success).toBeTrue();
  expect(data.results).toBeArray();
  const extractedText = data.results.map((r) => r.text).join(' ');
  console.log(`Extracted text: "${extractedText}"`);
  // Check that we got some text back
  expect(data.results.length).toBeGreaterThan(0);
  // Check that at least some of the expected text was found
  const normalizedExtracted = extractedText.toLowerCase().replace(/\s+/g, '');
  const normalizedExpected = testText.toLowerCase().replace(/\s+/g, '');
  const hasPartialMatch =
    normalizedExtracted.includes('hello') ||
    normalizedExtracted.includes('world') ||
    normalizedExtracted.includes('12345');
  expect(hasPartialMatch).toBeTrue();
 });
 // File upload OCR test
 tap.test('should perform OCR via file upload', async () => {
  const testText = 'Invoice Number 98765';
  console.log(`Creating test image with text: "${testText}"`);
  const imageBase64 = createTestImage(testText);
  const imageBuffer = Buffer.from(imageBase64, 'base64');
  const formData = new FormData();
  const blob = new Blob([imageBuffer], { type: 'image/png' });
  formData.append('img', blob, 'test.png');
  const response = await fetch(`${PADDLEOCR_URL}/ocr/upload`, {
    method: 'POST',
    body: formData,
  });
  expect(response.ok).toBeTrue();
  const data: IOCRResponse = await response.json();
  expect(data.success).toBeTrue();
  expect(data.results).toBeArray();
  const extractedText = data.results.map((r) => r.text).join(' ');
  console.log(`Extracted text: "${extractedText}"`);
  // Check that we got some text back
  expect(data.results.length).toBeGreaterThan(0);
 });
 // OCR result structure test
 tap.test('should return proper OCR result structure', async () => {
  const testText = 'Test 123';
  const imageBase64 = createTestImage(testText);
  const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ image: imageBase64 }),
  });
  const data: IOCRResponse = await response.json();
  if (data.results.length > 0) {
    const result = data.results[0];
    // Check result has required fields
    expect(result.text).toBeTypeofString();
    expect(result.confidence).toBeTypeofNumber();
    expect(result.box).toBeArray();
    // Check bounding box structure (4 points, each with x,y)
    expect(result.box.length).toEqual(4);
    for (const point of result.box) {
      expect(point.length).toEqual(2);
      expect(point[0]).toBeTypeofNumber();
      expect(point[1]).toBeTypeofNumber();
    }
    // Confidence should be between 0 and 1
    expect(result.confidence).toBeGreaterThan(0);
    expect(result.confidence).toBeLessThanOrEqual(1);
    console.log(`Result structure valid:`);
    console.log(`  Text: "${result.text}"`);
    console.log(`  Confidence: ${(result.confidence * 100).toFixed(1)}%`);
    console.log(`  Box: ${JSON.stringify(result.box)}`);
  }
 });
 // Test with actual invoice if available
 const invoiceDir = path.join(process.cwd(), '.nogit/invoices');
 if (fs.existsSync(invoiceDir)) {
  const pdfFiles = fs.readdirSync(invoiceDir).filter((f) => f.endsWith('.pdf'));
  if (pdfFiles.length > 0) {
    const testPdf = pdfFiles[0];
    tap.test(`should extract text from invoice: ${testPdf}`, async () => {
      const pdfPath = path.join(invoiceDir, testPdf);
      console.log(`Converting ${testPdf} to image...`);
      const imageBase64 = convertPdfToImage(pdfPath);
      console.log(`Image size: ${(imageBase64.length / 1024).toFixed(1)} KB`);
      const startTime = Date.now();
      const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ image: imageBase64 }),
      });
      const endTime = Date.now();
      const elapsedMs = endTime - startTime;
      expect(response.ok).toBeTrue();
      const data: IOCRResponse = await response.json();
      expect(data.success).toBeTrue();
      console.log(`OCR completed in ${(elapsedMs / 1000).toFixed(2)}s`);
      console.log(`Found ${data.results.length} text regions`);
      // Print first 10 results
      const preview = data.results.slice(0, 10);
      console.log(`\nFirst ${preview.length} results:`);
      for (const result of preview) {
        console.log(`  [${(result.confidence * 100).toFixed(0)}%] ${result.text}`);
      }
      if (data.results.length > 10) {
        console.log(`  ... and ${data.results.length - 10} more`);
      }
      // Should find text in an invoice
      expect(data.results.length).toBeGreaterThan(5);
    });
  }
 }
 // Error handling test
 tap.test('should handle invalid base64 gracefully', async () => {
  const response = await fetch(`${PADDLEOCR_URL}/ocr`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ image: 'not-valid-base64!!!' }),
  });
  const data: IOCRResponse = await response.json();
  // Should return success: false with error message
  expect(data.success).toBeFalse();
  expect(data.error).toBeTypeofString();
  console.log(`Error handling works: ${data.error}`);
 });
 export default tap.start();
Author	SHA1	Message	Date
Juergen Kunz	26d2de824f	v1.15.1 Some checks failed Docker (tags) / security (push) Successful in 22s Details Docker (tags) / test (push) Failing after 7m38s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-20 03:19:58 +00:00
Juergen Kunz	969d21c51a	fix(tests): enable progress events in invoice tests and bump @push.rocks/smartagent devDependency to ^1.5.4	2026-01-20 03:19:58 +00:00
Juergen Kunz	da2b827ba3	chore: update smartagent to v1.5.2 (streaming support for native tool calling)	2026-01-20 02:55:28 +00:00
Juergen Kunz	9bc1f74978	feat(test): enable native tool calling for GPT-OSS invoice extraction - Update smartai to v0.13.2 (native tool calling support) - Update smartagent to v1.5.1 (useNativeToolCalling option) - Enable think: true for GPT-OSS reasoning mode in Ollama config - Enable useNativeToolCalling: true in DualAgentOrchestrator - Simplify driver system message (native tools don't need XML instructions) Native tool calling uses Ollama's built-in Harmony format parser instead of requiring XML generation, which is more efficient for GPT-OSS models.	2026-01-20 02:51:52 +00:00
Juergen Kunz	cf282b2437	v1.15.0 Some checks failed Docker (tags) / security (push) Successful in 29s Details Docker (tags) / test (push) Failing after 7m38s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-20 01:17:41 +00:00
Juergen Kunz	77d57e80bd	feat(tests): integrate SmartAi/DualAgentOrchestrator into extraction tests and add JSON self-validation	2026-01-20 01:17:41 +00:00
Juergen Kunz	b202e024a4	v1.14.3 Some checks failed Docker (tags) / security (push) Successful in 29s Details Docker (tags) / test (push) Failing after 1m46s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-20 00:55:24 +00:00
Juergen Kunz	2210611f70	fix(repo): no changes detected in the diff; no files modified and no release required	2026-01-20 00:55:24 +00:00
Juergen Kunz	d8bdb18841	fix(test): add JSON validation and retry logic to invoice extraction - Add tryExtractJson function to validate JSON before accepting - Use orchestrator.continueTask() to request correction if JSON is invalid - Retry up to 2 times for malformed JSON responses - Remove duplicate parseJsonToInvoice function	2026-01-20 00:45:30 +00:00
Juergen Kunz	d384c1d79b	feat(tests): integrate smartagent DualAgentOrchestrator with streaming support - Update test.invoices.nanonets.ts to use DualAgentOrchestrator for JSON extraction - Enable streaming token callback for real-time progress visibility - Add markdown caching to avoid re-running Nanonets OCR for cached files - Update test.bankstatements.minicpm.ts and test.invoices.minicpm.ts with streaming - Update dependencies to @push.rocks/smartai@0.11.1 and @push.rocks/smartagent@1.2.8	2026-01-20 00:39:36 +00:00
Juergen Kunz	6bd672da61	v1.14.2 Some checks failed Docker (tags) / security (push) Successful in 28s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-19 21:28:26 +00:00
Juergen Kunz	44d6dc3336	fix(readme): update README to document Nanonets-OCR2-3B (replaces Nanonets-OCR-s), adjust VRAM and context defaults, expand feature docs, and update examples/test command	2026-01-19 21:28:26 +00:00
Juergen Kunz	d1ff95bd94	v1.14.1 Some checks failed Docker (tags) / security (push) Successful in 26s Details Docker (tags) / test (push) Failing after 39s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-19 21:19:37 +00:00
Juergen Kunz	09770d3177	fix(extraction): improve JSON extraction prompts and model options for invoice and bank statement tests	2026-01-19 21:19:37 +00:00
Juergen Kunz	235aa1352b	v1.14.0 Some checks failed Docker (tags) / security (push) Successful in 29s Details Docker (tags) / test (push) Failing after 39s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-19 21:05:51 +00:00
Juergen Kunz	08728ada4d	feat(docker-images): add vLLM-based Nanonets-OCR2-3B image, Qwen3-VL Ollama image and refactor build/docs/tests to use new runtime/layout	2026-01-19 21:05:51 +00:00
Juergen Kunz	b58bcabc76	update	2026-01-19 11:51:23 +00:00
Juergen Kunz	6dbd06073b	v1.13.2 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 23:00:24 +00:00
Juergen Kunz	ae28a64902	fix(tests): stabilize OCR extraction tests and manage GPU containers	2026-01-18 23:00:24 +00:00
Juergen Kunz	09ea7440e8	update	2026-01-18 15:54:16 +00:00
Juergen Kunz	177e87d3b8	v1.13.1 Some checks failed Docker (tags) / security (push) Successful in 24s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 13:58:26 +00:00
Juergen Kunz	17ea7717eb	fix(image_support_files): remove PaddleOCR-VL server scripts from image_support_files	2026-01-18 13:58:26 +00:00
Juergen Kunz	bd5bb5d874	v1.13.0 Some checks failed Docker (tags) / security (push) Successful in 29s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 13:56:46 +00:00
Juergen Kunz	d91df70fff	feat(tests): revamp tests and remove legacy Dockerfiles: adopt JSON/consensus workflows, switch MiniCPM model, and delete deprecated Docker/test variants	2026-01-18 13:56:46 +00:00
Juergen Kunz	d6c97a9625	v1.12.0 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 57s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 11:26:38 +00:00
Juergen Kunz	76b21f1f7b	feat(tests): switch vision tests to multi-query extraction (count then per-row/field queries) and add logging/summaries	2026-01-18 11:26:38 +00:00
Juergen Kunz	4c368dfef9	v1.11.0 Some checks failed Docker (tags) / security (push) Successful in 29s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 04:50:57 +00:00
Juergen Kunz	e76768da55	feat(vision): process pages separately and make Qwen3-VL vision extraction more robust; add per-page parsing, safer JSON handling, reduced token usage, and multi-query invoice extraction	2026-01-18 04:50:57 +00:00
Juergen Kunz	63d72a52c9	update	2026-01-18 04:28:57 +00:00
Juergen Kunz	386122c8c7	v1.10.1 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 04:17:30 +00:00
Juergen Kunz	7c8f10497e	fix(tests): improve Qwen3-VL invoice extraction test by switching to non-stream API, adding model availability/pull checks, simplifying response parsing, and tightening model options	2026-01-18 04:17:30 +00:00
Juergen Kunz	9f9ec0a671	v1.10.0 Some checks failed Docker (tags) / security (push) Successful in 32s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 03:35:06 +00:00
Juergen Kunz	3780105c6f	feat(vision): add Qwen3-VL vision model support with Dockerfile and tests; improve invoice OCR conversion and prompts; simplify extraction flow by removing consensus voting	2026-01-18 03:35:05 +00:00
Juergen Kunz	d237ad19f4	v1.9.0 Some checks failed Docker (tags) / security (push) Successful in 33s Details Docker (tags) / test (push) Failing after 39s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 02:53:24 +00:00
Juergen Kunz	7652a2df52	feat(tests): add Ministral 3 vision tests and improve invoice extraction pipeline to use Ollama chat schema, sanitization, and multi-page support	2026-01-18 02:53:24 +00:00
Juergen Kunz	b316d98f24	v1.8.0 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 41s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-18 00:11:17 +00:00
Juergen Kunz	f0d88fcbe0	feat(paddleocr-vl): add structured HTML output and table parsing for PaddleOCR-VL, update API, tests, and README	2026-01-18 00:11:17 +00:00
Juergen Kunz	0d8a1ebac2	v1.7.1 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 39s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-17 23:13:47 +00:00
Juergen Kunz	5a311dca2d	fix(docker): standardize Dockerfile and entrypoint filenames; add GPU-specific Dockerfiles and update build and test references	2026-01-17 23:13:47 +00:00
Juergen Kunz	ab288380f1	v1.7.0 Some checks failed Docker (tags) / security (push) Successful in 30s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-17 21:50:09 +00:00
Juergen Kunz	30c73b24c1	feat(tests): use Qwen2.5 (Ollama) for invoice extraction tests and add helpers for model management; normalize dates and coerce numeric fields	2026-01-17 21:50:09 +00:00
Juergen Kunz	311e7a8fd4	v1.6.0 Some checks failed Docker (tags) / security (push) Successful in 32s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-17 20:22:23 +00:00
Juergen Kunz	80e6866442	feat(paddleocr-vl): add PaddleOCR-VL full pipeline Docker image and API server, plus integration tests and docker helpers	2026-01-17 20:22:23 +00:00
Juergen Kunz	addae20cbd	v1.5.0 Some checks failed Docker (tags) / security (push) Successful in 31s Details Docker (tags) / test (push) Failing after 40s Details Docker (tags) / release (push) Has been skipped Details Docker (tags) / metadata (push) Has been skipped Details	2026-01-17 16:57:26 +00:00
Juergen Kunz	0482c35b69	feat(paddleocr-vl): add PaddleOCR-VL GPU Dockerfile, pin vllm, update CPU image deps, and improve entrypoint and tests	2026-01-17 16:57:26 +00:00
Juergen Kunz	15ac1fcf67	update	2026-01-16 16:21:44 +00:00