Docker Ollama

create-folder-structure.sh

sudo mkdir -p /data/ollama-models
sudo chown $USER:$USER /data/ollama-models
mkdir -p ~/docker/ollama

docker-compose.yml

# nano ~/docker/ollama/docker-compose.yml
services:
  ollama:
    image: ollama/ollama:latest
    container_name: ollama
    restart: always
    ports:
      - "11434:11434"
    environment:
      - OLLAMA_HOST=0.0.0.0
      - OLLAMA_NO_CLOUD=1
    volumes:
      - /data/ollama-models:/root/.ollama/models
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    container_name: open-webui
    restart: always
    ports:
      - "3000:8080"
    environment:
      - OLLAMA_BASE_URL=http://ollama:11434
    volumes:
      - open-webui:/app/backend/data
    depends_on:
      - ollama

volumes:
  open-webui:

examine-gpu-access.sh

docker run --rm --gpus all nvidia/cuda:12.0-base-ubuntu22.04 nvidia-smi

examine-gpu-detection.sh

docker exec -it ollama ollama list
docker logs ollama | grep -i gpu

install-docker.sh

curl -fsSL https://get.docker.com | sh  
sudo usermod -aG docker $USER  
newgrp docker

install-nvidia-toolkit.sh

curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
sudo apt update
sudo apt install -y nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

start-ollama.sh

cd ~/docker/ollama
docker compose up -d
docker compose ps