GitHub - katanaml/sparrow: Structured data extraction and instruction calling with ML, LLM and Vision LLM

# 1. Install pyenv and Python 3.12.10
pyenv install 3.12.10
pyenv global 3.12.10

# 2. Create virtual environment
python -m venv .env_sparrow_parse
source .env_sparrow_parse/bin/activate  # Linux/Mac
# or .env_sparrow_parse\Scripts\activate  # Windows

# 3. Install Sparrow Parse pipeline
git clone https://github.com/katanaml/sparrow.git
cd sparrow/sparrow-ml/llm
pip install -r requirements_sparrow_parse.txt

# 4. For macOS: Install poppler for PDF processing
brew install poppler

# 5. Start the API server
python api.py

# Extract data from a bonds table
./sparrow.sh '[{"instrument_name":"str", "valuation":0}]' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "data/bonds_table.png"

{
  "data": [
    {"instrument_name": "UNITS BLACKROCK...", "valuation": 19049},
    {"instrument_name": "UNITS ISHARES...", "valuation": 83488}
  ],
  "valid": "true"
}

# 1. Clone repository
git clone https://github.com/katanaml/sparrow.git
cd sparrow

brew install poppler  # Required for PDF processing

sudo apt-get install poppler-utils libpoppler-cpp-dev

# Test installation
python api.py --port 8002
# Visit http://localhost:8002/api/v1/sparrow-llm/docs

# Extract all data from bank statement
./sparrow.sh "*" \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "data/bank_statement.pdf"

# Extract structured data from financial table
./sparrow.sh '[{"instrument_name":"str", "valuation":0}]' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "data/bonds_table.png"

# Extract invoice with cropping for better accuracy
./sparrow.sh "*" \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --crop-size 60 \
  --file-path "data/invoice.pdf"

# Process multi-page PDF with structured output per page
./sparrow.sh '{"table": [{"description": "str", "latest_amount": 0, "previous_amount": 0}]}' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "data/financial_report.pdf" \
  --debug-dir "debug/"

# Instruction-based processing
./sparrow.sh "instruction: do arithmetic operation, payload: 2+2=" \
  --pipeline "sparrow-instructor" \
  --options mlx \
  --options lmstudio-community/Mistral-Small-3.2-24B-Instruct-2506-8bit

# Instruction processing with document input
./sparrow.sh "check if business entity Chapman, Kim and Green is invoice issuing party" 
  --pipeline "sparrow-parse" 
  --instruction 
  --options mlx --options lmstudio-community/Mistral-Small-3.2-24B-Instruct-2506-8bit 
  --file-path "invoice_1.jpg"

The result of 2 + 2 is:

4

# Function calling example
./sparrow.sh assistant --pipeline "stocks" --query "Oracle"

{
  "company": "Oracle Corporation",
  "ticker": "ORCL"
}

The stock price of the Oracle Corporation is 186.3699951171875. USD

./sparrow.sh "<JSON_SCHEMA>" --pipeline "<PIPELINE>" [OPTIONS] --file-path "<FILE>"

# MLX Backend (Apple Silicon)
./sparrow.sh '[{"instrument_name":"str", "valuation":0}]' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "data/bonds_table.png"

# Hugging Face Cloud GPU
--options huggingface --options your-space/model-name

# Additional flags
--options tables_only        # Extract only tables
--options validation_off     # Disable schema validation
--options apply_annotation   # Include bounding boxes
--page-type financial_table  # Classify page type

# Instruction-based processing
./sparrow.sh "instruction: do arithmetic operation, payload: 2+2=" \
  --pipeline "sparrow-instructor" \
  --options mlx \
  --options lmstudio-community/Mistral-Small-3.2-24B-Instruct-2506-8bit

# Multi-page PDF with page classification
./sparrow.sh "*" \
  --page-type invoice \
  --page-type table \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "multi_page.pdf"

# Handle missing fields with null values
./sparrow.sh '[{"required_field":"str", "optional_field":"str or null"}]' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --file-path "document.png"

# Table extraction with cropping
./sparrow.sh '*' \
  --pipeline "sparrow-parse" \
  --options mlx \
  --options mlx-community/Qwen2.5-VL-72B-Instruct-4bit \
  --options tables_only \
  --crop-size 100 \
  --file-path "scan.pdf"

# Instruction execution
./sparrow.sh "check if business entity Chapman, Kim and Green is invoice issuing party" 
  --pipeline "sparrow-parse" 
  --instruction 
  --options mlx --options lmstudio-community/Mistral-Small-3.2-24B-Instruct-2506-8bit 
  --file-path "invoice_1.jpg"

# Field validation
./sparrow.sh "tax_id,shipment_code,total_gross_worth" 
  --pipeline "sparrow-parse" 
  --validation 
  --options mlx --options lmstudio-community/Mistral-Small-3.2-24B-Instruct-2506-8bit 
  --file-path "invoice_1.jpg"

{
  "tax_id": true,
  "shipment_code": false,
  "total_gross_worth": true
}

# Default port (8002)
python api.py

# Custom port
python api.py --port 8001

# Multiple instances
python api.py --port 8002 &  # Sparrow Parse
python api.py --port 8003 &  # Instructor

curl -X POST 'http://localhost:8002/api/v1/sparrow-llm/inference' \
  -H 'Content-Type: multipart/form-data' \
  -F 'query=[{"field_name":"str", "amount":0}]' \
  -F 'pipeline=sparrow-parse' \
  -F 'options=mlx,mlx-community/Qwen2.5-VL-72B-Instruct-4bit' \
  -F 'file=@document.pdf'

curl -X POST 'http://localhost:8002/api/v1/sparrow-llm/instruction-inference' \
  -H 'Content-Type: application/x-www-form-urlencoded' \
  -d 'query=instruction: analyze data, payload: {...}' \
  -d 'pipeline=sparrow-instructor' \
  -d 'options=mlx,mlx-community/Qwen2.5-VL-72B-Instruct-4bit'

# Start agent server
cd sparrow-ml/agents
python api.py --port 8001

# Process medical prescriptions
curl -X POST 'http://localhost:8001/api/v1/sparrow-agents/execute/file' \
  -F 'agent_name=medical_prescriptions' \
  -F 'extraction_params={"sparrow_key":"123456"}' \
  -F 'file=@prescription.pdf'

# Reduce memory usage
--crop-size 100        # Crop large images
--options tables_only  # Process only tables

# For large PDFs
--debug-dir ./temp     # Monitor processing
# Split large PDFs manually if needed