GitHub - ccheshirecat/nomad-driver-ch: Task driver plugin for running cloud hypervisor virtual machines with the nomad orchestrator. Supports cloud init and VFIO passthrough.

job "web-server" {
  datacenters = ["dc1"]
  type = "service"

  group "web" {
    task "nginx" {
      driver = "ch"

      config {
        image = "/var/lib/images/alpine-nginx.img"

        network_interface {
          bridge {
            name = "br0"
            static_ip = "192.168.1.100"
          }
        }
      }

      resources {
        cpu    = 1000
        memory = 512
      }
    }
  }
}
```## 📦 Installation

### 1. Install Dependencies

**Cloud Hypervisor:**
```bash
# Download and install Cloud Hypervisor v48+
wget https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v48.0/cloud-hypervisor-static
sudo mv cloud-hypervisor-static /usr/local/bin/cloud-hypervisor
sudo chmod +x /usr/local/bin/cloud-hypervisor

# Install ch-remote for VM management
wget https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v48.0/ch-remote-static
sudo mv ch-remote-static /usr/local/bin/ch-remote
sudo chmod +x /usr/local/bin/ch-remote

# Optional: ensure binaries are discoverable
export PATH="/usr/local/bin:$PATH"

# Install virtiofsd for filesystem sharing
sudo apt-get install virtiofsd  # Ubuntu/Debian
# or
sudo yum install virtiofsd      # RHEL/CentOS

# Create bridge interface
sudo ip link add br0 type bridge
sudo ip addr add 192.168.1.1/24 dev br0
sudo ip link set br0 up

# Configure bridge persistence (systemd-networkd)
cat > /etc/systemd/network/br0.netdev << EOF
[NetDev]
Name=br0
Kind=bridge
EOF

cat > /etc/systemd/network/br0.network << EOF
[Match]
Name=br0

[Network]
IPForward=yes
Address=192.168.1.1/24
EOF

sudo systemctl restart systemd-networkd

# Download latest release
wget https://github.com/ccheshirecat/nomad-driver-ch/releases/latest/download/nomad-driver-ch
sudo mv nomad-driver-ch /opt/nomad/plugins/
sudo chmod +x /opt/nomad/plugins/nomad-driver-ch

git clone https://github.com/ccheshirecat/nomad-driver-ch.git
cd nomad-driver-ch
go build -o nomad-driver-ch .
sudo mv nomad-driver-ch /opt/nomad/plugins/

# /etc/nomad.d/client.hcl
client {
  enabled = true

  plugin "nomad-driver-ch" {
    config {
      # Cloud Hypervisor configuration
      cloud_hypervisor {
        bin = "/usr/bin/cloud-hypervisor"
        remote_bin = "/usr/bin/ch-remote"
        virtiofsd_bin = "/usr/libexec/virtiofsd"
        default_kernel = "/boot/vmlinuz"
        default_initramfs = "/boot/initramfs.img"
      }
        disable_alloc_mounts = false

      # Network configuration
      network {
        bridge = "br0"
        subnet_cidr = "192.168.1.0/24"
        gateway = "192.168.1.1"
        ip_pool_start = "192.168.1.100"
        ip_pool_end = "192.168.1.200"
      }

      # Allowed image paths for security
      image_paths = ["/var/lib/images", "/opt/vm-images"]
    }
  }
}

sudo systemctl restart nomad

nomad node status -self | grep ch

plugin "nomad-driver-ch" {
  config {
    # Cloud Hypervisor binaries
    cloud_hypervisor {
      bin = "/usr/bin/cloud-hypervisor"           # Cloud Hypervisor binary path
      remote_bin = "/usr/bin/ch-remote"           # ch-remote binary path
      virtiofsd_bin = "/usr/libexec/virtiofsd"        # virtiofsd binary path
      default_kernel = "/boot/vmlinuz"            # Default kernel for VMs
      default_initramfs = "/boot/initramfs.img"   # Default initramfs for VMs
      firmware = "/usr/share/qemu/OVMF.fd"        # UEFI firmware (optional)
      seccomp = "true"                            # Enable seccomp filtering
      log_file = "/var/log/cloud-hypervisor.log" # VM log file path
    }

    # Network configuration
    network {
      bridge = "br0"                      # Bridge interface name
      subnet_cidr = "192.168.1.0/24"      # Subnet for VMs
      gateway = "192.168.1.1"             # Gateway IP address
      ip_pool_start = "192.168.1.100"     # IP pool start range
      ip_pool_end = "192.168.1.200"       # IP pool end range
      tap_prefix = "tap"                  # TAP interface prefix
    }

    # VFIO device passthrough (not yet implemented)
    # vfio {
    #   allowlist = ["10de:*", "8086:0d26"]  # PCI device allowlist
    #   iommu_address_width = 48              # IOMMU address width
    #   pci_segments = 1                      # Number of PCI segments
    # }

    # Security and paths
    data_dir = "/opt/nomad/data"              # Nomad data directory
    image_paths = [                           # Allowed image paths
      "/var/lib/images",
      "/opt/vm-images",
      "/mnt/shared-storage"
    ]
  }
}

job "basic-vm" {
  datacenters = ["dc1"]

  group "app" {
    task "vm" {
      driver = "ch"

      config {
        image = "/var/lib/images/ubuntu-22.04.img"
        hostname = "app-server"

        # REQUIRED: kernel and initramfs (Cloud Hypervisor has no bootloader)
        kernel = "/boot/vmlinuz-5.15.0"
        initramfs = "/boot/initramfs-5.15.0.img"
        cmdline = "console=ttyS0 root=/dev/vda1"
      }

      resources {
        cpu    = 2000  # 2 CPU cores
        memory = 2048  # 2GB RAM
      }

      # Optional: allow sandbox/CI environments without binaries
      # skip_binary_validation = true
    }
  }
}

job "custom-vm" {
  datacenters = ["dc1"]

  group "web" {
    task "nginx" {
      driver = "ch"

      config {
        image = "/var/lib/images/alpine.img"
        hostname = "nginx-server"

        # Cloud-init user data
        user_data = "/etc/cloud-init/nginx-setup.yml"

        # Default user configuration
        default_user_password = "secure123"
        default_user_authorized_ssh_key = "ssh-rsa AAAAB3NzaC1yc2E..."

        # Custom commands to run
        cmds = [
          "apk add --no-cache nginx",
          "rc-service nginx start",
          "rc-update add nginx default"
        ]
      }

      resources {
        cpu    = 1000
        memory = 512
      }
    }
  }
}

job "database" {
  datacenters = ["dc1"]

  group "db" {
    task "postgres" {
      driver = "ch"

      config {
        image = "/var/lib/images/postgres-14.img"
        hostname = "postgres-primary"

        # Enable thin copy for faster startup
        use_thin_copy = true

        # Network configuration with static IP
        network_interface {
          bridge {
            name = "br0"
            static_ip = "192.168.1.50"
            gateway = "192.168.1.1"
            netmask = "24"
            dns = ["8.8.8.8", "1.1.1.1"]
          }
        }

        # Custom timezone
        timezone = "America/New_York"
      }

      resources {
        cpu    = 4000  # 4 CPU cores
        memory = 8192  # 8GB RAM
      }

      # Mount shared storage
      volume_mount {
        volume      = "postgres-data"
        destination = "/var/lib/postgresql"
      }
    }
  }

  volume "postgres-data" {
    type      = "host"
    source    = "postgres-data"
    read_only = false
  }
}

job "ml-workload" {
  datacenters = ["dc1"]

  group "gpu" {
    task "training" {
      driver = "ch"


      config {
        image = "/var/lib/images/cuda-ubuntu.img"
        hostname = "ml-trainer"

        # VFIO GPU passthrough (not yet implemented)
        # vfio_devices = ["10de:2204"]  # NVIDIA RTX 3080
      }

      resources {
        cpu      = 8000  # 8 CPU cores
        memory   = 16384 # 16GB RAM
        device "nvidia/gpu" {
          count = 1
        }
      }
    }
  }
}

config {
  network_interface {
    bridge {
      name = "br0"
      # IP will be allocated from pool automatically
    }
  }
}

config {
  network_interface {
    bridge {
      name = "br0"
      static_ip = "192.168.1.100"
      gateway = "192.168.1.1"
      netmask = "24"
      dns = ["8.8.8.8", "1.1.1.1"]
    }
  }
}

config {
  network_interface {
    bridge {
      name = "br0"
      ports = ["web", "api"]  # Reference port labels from network block
    }
  }
}

network {
  port "web" {
    static = 80
  }
  port "api" {
    static = 8080
  }
}

config {
  user_data = "/etc/cloud-init/web-server.yml"
}

#cloud-config
packages:
  - nginx
  - curl
  - htop

runcmd:
  - systemctl enable nginx
  - systemctl start nginx
  - ufw allow 80
  - ufw --force enable

write_files:
  - path: /var/www/html/index.html
    content: |
      <!DOCTYPE html>
      <html>
        <head><title>Hello from Nomad VM</title></head>
        <body><h1>VM deployed via Nomad Cloud Hypervisor driver!</h1></body>
      </html>
    permissions: '0644'

config {
  user_data = <<EOF
#cloud-config
package_update: true
packages:
  - docker.io
runcmd:
  - systemctl enable docker
  - systemctl start docker
  - docker run -d -p 80:80 nginx:alpine
EOF
}

config {
  default_user_password = "secure-password"
  default_user_authorized_ssh_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQAB..."
}

config {
  # Commands run during boot process
  cmds = [
    "apt-get update",
    "apt-get install -y docker.io",
    "systemctl enable docker"
  ]
}

network_interface {
  bridge {
    name  = "br0"
    static_ip = "192.168.1.100"
    ports = ["http", "https"]
  }
}

network_interface {
  bridge {
    name  = "br0"
    ports = ["http", "https"]  # Port forwarding works with DHCP!
  }
}

config {
  image = "/var/lib/images/base-ubuntu.img"
  use_thin_copy = true
}

job "shared-storage" {
  group "app" {
    volume "shared-data" {
      type   = "host"
      source = "app-data"
    }

    task "processor" {
      driver = "ch"

      config {
        image = "/var/lib/images/data-processor.img"
      }

      volume_mount {
        volume      = "shared-data"
        destination = "/app/data"
        read_only   = false
      }

      resources {
        cpu    = 2000
        memory = 4096
      }
    }
  }
}

plugin "nomad-driver-ch" {
  config {
    vfio {
      # Allowlist specific devices (vendor:device format)
      allowlist = [
        "10de:*",      # All NVIDIA GPUs
        "8086:0d26",   # Intel specific device
        "1002:67df"    # AMD Radeon RX 480
      ]
      iommu_address_width = 48  # Default: 48
      pci_segments = 1          # Default: 1
    }
  }
}

job "ai-training" {
  datacenters = ["dc1"]

  constraint {
    attribute = "${node.unique.name}"
    value     = "gpu-node-1"
  }

  group "training" {
    task "model-training" {
      driver = "ch"

      config {
        image = "/var/lib/images/cuda-pytorch.img"
        
        # Pass through NVIDIA RTX 3080 (GPU + Audio controller)
        vfio_devices = ["0000:01:00.0", "0000:01:00.1"]
      }

      resources {
        cpu    = 8000
        memory = 32768
      }
    }
  }
}

## 📊 Monitoring

### Resource Statistics

The driver provides real-time VM resource statistics:

```bash
# View allocation statistics
nomad alloc status <alloc-id>

# Monitor resource usage
nomad alloc logs -f <alloc-id> <task-name>

task "web-server" {
 driver = "ch"

  config {
    image = "/var/lib/images/nginx.img"
    network_interface {
      bridge {
        name = "br0"
        static_ip = "192.168.1.100"
      }
    }
  }

  service {
    name = "web"
    port = "http"

    check {
      type     = "http"
      path     = "/"
      interval = "30s"
      timeout  = "5s"
      address_mode = "alloc"
    }
  }
}

# 1. Verify image format
qemu-img info /path/to/image.img

# 2. Check image paths configuration
nomad agent-info | grep -A 10 virt

# 3. Validate kernel/initramfs paths
ls -la /boot/vmlinuz* /boot/initramfs*

# 4. Test Cloud Hypervisor directly
cloud-hypervisor --kernel /boot/vmlinuz --disk path=/path/to/image.img

# 1. Check bridge configuration
ip link show br0
brctl show br0

# 2. Verify TAP interface creation
ip link show | grep tap

# 3. Test bridge connectivity
ping 192.168.1.1  # Gateway IP

# 4. Check iptables rules
iptables -L -v -n

log_level = "DEBUG"
enable_debug = true

# Check Cloud Hypervisor processes
ps aux | grep cloud-hypervisor

# Inspect VM via ch-remote
ch-remote --api-socket /path/to/api.sock info

# Monitor VM console output
tail -f /opt/nomad/data/alloc/<alloc-id>/<task>/serial.log

config {
  # Required: VM disk image path
  image = "/path/to/vm-image.img"

  # Optional: VM hostname
  hostname = "my-vm-host"

  # Optional: Operating system variant
  os {
    arch    = "x86_64"        # CPU architecture
    machine = "q35"           # Machine type
    variant = "ubuntu20.04"   # OS variant
  }

  # Optional: Cloud-init user data
  user_data = "/path/to/user-data.yml"  # File path
  # OR
  user_data = <<EOF           # Inline YAML
  #cloud-config
  packages:
    - nginx
  EOF

  # Optional: Timezone configuration
  timezone = "America/New_York"

  # Optional: Custom commands to run
  cmds = [
    "apt-get update",
    "systemctl enable nginx"
  ]

  # Optional: Default user configuration
  default_user_authorized_ssh_key = "ssh-rsa AAAAB3..."
  default_user_password = "secure-password"

  # Optional: Storage configuration
  use_thin_copy = true        # Enable thin provisioning

  # Optional: Cloud Hypervisor specific
  kernel = "/boot/custom-kernel"      # Custom kernel path
  initramfs = "/boot/custom-initrd"   # Custom initramfs path
  cmdline = "console=ttyS0 quiet"     # Kernel command line

  # Optional: Network interface configuration
  network_interface {
    bridge {
      name = "br0"                    # Bridge name (required)
      ports = ["web", "api"]          # Port labels to expose
      static_ip = "192.168.1.100"     # Static IP address
      gateway = "192.168.1.1"         # Custom gateway
      netmask = "24"                  # Subnet mask (CIDR)
      dns = ["8.8.8.8", "1.1.1.1"]   # Custom DNS servers
    }
  }

  # Optional: VFIO device passthrough (coming very soon!)
  # vfio_devices = ["10de:2204"]  # PCI device IDs

  # Optional: USB device passthrough
  usb_devices = ["046d:c52b"]   # USB vendor:product IDs
}

resources {
  cpu    = 2000    # CPU shares (1 core = 1000)
  memory = 2048    # Memory in MB

  # Optional: GPU devices
  device "nvidia/gpu" {
    count = 1
    constraint {
      attribute = "${device.attr.compute_capability}"
      operator  = ">="
      value     = "6.0"
    }
  }
}

# Clone repository
git clone https://github.com/ccheshirecat/nomad-driver-ch.git
cd nomad-driver-ch

# Install dependencies
go mod download

# Run tests
go test ./...

# Build binary
go build -o nomad-driver-ch .

# Install plugin
sudo cp nomad-driver-ch /opt/nomad/plugins/

# Requires Cloud Hypervisor installation
sudo go test -v ./virt/... -run Integration