GitHub - mixpeek/multimodal-inference-server: Production-grade Rust inference server for multimodal models (image + text → streamed text), with OpenAI-compatible APIs and high-throughput GPU scheduling.

# Clone the repository
git clone https://github.com/mixpeek/multimodal-inference-server.git
cd vlm-inference-server

# Build the project (Release mode for best performance)
cargo build --release

# Start the worker (downloads 14GB model on first run)
./target/release/vlm-worker --host 0.0.0.0 --port 50051 &

# Start the gateway (HTTP API)
./target/release/vlm-gateway --host 0.0.0.0 --port 8080 &

# Send a chat completion request
curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "vlm-prod",
    "messages": [{"role": "user", "content": "Hello, how are you?"}],
    "max_tokens": 20
  }'

┌─────────────┐          ┌──────────┐          ┌─────────────┐
│   Client    │          │ Gateway  │          │   Worker    │
│  (HTTP)     │─────────▶│ (HTTP)   │─────────▶│  (gRPC)     │
│             │◀─────────│          │◀─────────│             │
└─────────────┘          └──────────┘          └─────────────┘
                               │                       │
                               │                       ▼
                               │              ┌─────────────────┐
                               │              │  Candle Engine  │
                               │              │  ┌───────────┐  │
                               │              │  │   CLIP    │  │
                               │              │  │  Vision   │  │
                               │              │  └───────────┘  │
                               │              │  ┌───────────┐  │
                               │              │  │  LLaMA-2  │  │
                               │              │  │    LLM    │  │
                               │              │  └───────────┘  │
                               │              └─────────────────┘
                               │
                               ▼
                      ┌──────────────────┐
                      │   Observability  │
                      │  Metrics, Logs   │
                      └──────────────────┘

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "vlm-prod",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "Explain quantum computing in simple terms."}
    ],
    "max_tokens": 100,
    "temperature": 0.7
  }'

curl -N -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "vlm-prod",
    "messages": [{"role": "user", "content": "Write a haiku about Rust"}],
    "max_tokens": 50,
    "stream": true
  }'

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "vlm-prod",
    "messages": [{
      "role": "user",
      "content": [
        {"type": "text", "text": "What'\''s in this image?"},
        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ..."}}
      ]
    }],
    "max_tokens": 100
  }'

vlm-inference-server/
├── crates/
│   ├── api_types/          # OpenAI-compatible API types
│   ├── proto/              # gRPC protocol definitions
│   ├── gateway/            # HTTP edge service
│   ├── worker/             # Inference worker service
│   ├── engine/             # ML engine trait definitions
│   ├── engine_adapters/
│   │   ├── mock_engine/    # Test mock
│   │   └── candle_engine/  # Candle ML implementation
│   ├── multimodal/         # Image preprocessing
│   ├── scheduler/          # Batching & admission control
│   ├── kv_cache/           # Key-value cache management
│   ├── sampling/           # Token sampling strategies
│   ├── common/             # Shared utilities
│   └── observability/      # Tracing & metrics
├── docs/                   # Documentation
├── examples/               # Usage examples
└── scripts/                # Helper scripts

# Run all tests
cargo test --workspace

# Run specific crate tests
cargo test --package vlm-candle-engine

# Run with logging
RUST_LOG=debug cargo test

# Run integration tests
cargo test --test '*'

# Debug build (faster compile, slower runtime)
cargo build

# Release build (optimized)
cargo build --release

# With specific features
cargo build --bin vlm-worker --features candle --release

# Check code style
cargo fmt --all -- --check
cargo clippy --all-targets --all-features

# Build image
docker build -t vlm-inference-server .

# Run container
docker run -p 8080:8080 -p 50051:50051 vlm-inference-server

# See k8s/ directory for full manifests
kubectl apply -f k8s/deployment.yaml
kubectl apply -f k8s/service.yaml

# See systemd/ directory for service files
sudo cp systemd/vlm-worker.service /etc/systemd/system/
sudo cp systemd/vlm-gateway.service /etc/systemd/system/
sudo systemctl enable --now vlm-worker vlm-gateway

Copyright 2026 VLM Inference Server Contributors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.