Qwen3.5-35B-A3B-Q4_K_M Docker Compose

docker-compose.yml

services:
  reasoning:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda
    container_name: qwen35-a3b-gguf
    restart: unless-stopped

    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility

    volumes:
      - ../models:/models

    ports:
      - "8080:8080"

    command: >
      -m /models/reasoning/Qwen3.5-35B-A3B/Qwen3.5-35B-A3B-Q4_K_M.gguf
      --host 0.0.0.0
      --port 8080
      --ctx-size 16384 
      --temp 1.0 
      --top-p 0.95 
      --top-k 20 
      --min-p 0.00