HAProxy and Docker Swarm

blog20191008-01.sh

$ apt update
$ apt install apt-transport-https ca-certificates curl gnupg2 software-properties-common
$ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
$ add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
$ apt update
$ apt install docker-ce docker-ce-cli containerd.io

blog20191008-02.sh

$ sudo docker swarm init --advertise-addr 10.64.137.201

Swarm initialized: current node (xvkdic6akgbx0k1odao9ce2yt) is now a manager.

To add a worker to this swarm, run the following command:

docker swarm join --token SWMTKN-1-67rcacnzj6ihnew1nd50gvhdsm087d4a2qgvgulxu62b7xbwep-37l1kvmfe309bp3fzvcaa4rq0 10.64.137.201:2377

To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.

blog20191008-03.sh

$ sudo docker swarm join --token [YOUR-SWARM-TOKEN] 10.64.137.201:2377

This node joined a swarm as a worker.

blog20191008-04.sh

$ sudo docker node ls

ID                          HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
xvkdic6akgbx0k1odao9ce2yt * dock1    Ready  Active       Leader         19.03.1
xaj6gfj9xkks3ybtjvsr44emh   dock2    Ready  Active                      19.03.1
q3wdfk3l8vinh227jwo99y3n7   dock3    Ready  Active                      19.03.1

blog20191008-05.sh

$ sudo docker node promote $(sudo docker node ls --filter role=worker --quiet)

Node xaj6gfj9xkks3ybtjvsr44emh promoted to a manager in the swarm.
Node q3wdfk3l8vinh227jwo99y3n7 promoted to a manager in the swarm.

blog20191008-06.sh

$ sudo docker network create --attachable --driver overlay apache-network

blog20191008-07.sh

$ sudo docker service create \
  --mode replicated \
  --replicas 0 \
  --name apache-Service \
  --network apache-network \
  --endpoint-mode dnsrr \
  httpd:2.4

blog20191008-08.cfg

global
    log          fd@2 local2
    chroot       /var/lib/haproxy
    pidfile      /var/run/haproxy.pid
    maxconn      4000
    user         haproxy
    group        haproxy
    stats socket /var/lib/haproxy/stats expose-fd listeners
    master-worker

blog20191008-09.cfg

resolvers docker
    nameserver dns1 127.0.0.11:53
    resolve_retries 3
    timeout resolve 1s
    timeout retry   1s
    hold other      10s
    hold refused    10s
    hold nx         10s
    hold timeout    10s
    hold valid      10s
    hold obsolete   10s

blog20191008-10.cfg

defaults
    timeout connect 10s
    timeout client 30s
    timeout server 30s
    log global
    mode http
    option httplog

blog20191008-11.cfg

frontend  fe_web
    bind *:80
    use_backend stat if { path -i /my-stats }
    default_backend be_apache_service 

backend be_apache_service
    balance roundrobin
    server-template apache- 6 apache-Service:80 check resolvers docker init-addr libc,none

backend be_apache_service_wrong_case
    balance roundrobin
    server-template apache- 6 apache-service:80 check resolvers docker init-addr libc,none

backend stat
    stats enable
    stats uri /my-stats
    stats refresh 15s
    stats show-legends
    stats show-node

blog20191008-12.sh

$ sudo docker service create \
  --mode replicated \
  --replicas 1 \
  --name haproxy-service \
  --network apache-network \
  --publish published=80,target=80,protocol=tcp,mode=ingress \
  --publish published=443,target=443,protocol=tcp,mode=ingress \
  --mount type=bind,src=/etc/haproxy/,dst=/etc/haproxy/,ro=true \
  --dns=127.0.0.11 \
  haproxytech/haproxy-debian:2.0

blog20191008-13.sh

$ sudo docker service logs --tail 20 haproxy-service

blog20191008-14.sh

$ sudo docker service rm haproxy-service

blog20191008-15.sh

$ sudo docker service create \
  --mode replicated \
  --replicas 1 \
  --name haproxy-service \
  --network apache-network \
  --publish published=80,target=80,protocol=tcp,mode=host \
  --publish published=443,target=443,protocol=tcp,mode=host \
  --mount type=bind,src=/etc/haproxy/,dst=/etc/haproxy/,ro=true \
  --dns=127.0.0.11 \
  haproxytech/haproxy-debian:2.0

blog20191008-16.sh

$ sudo apt install keepalived

blog20191008-17.conf

vrrp_script haproxy_container {
    script "/usr/bin/docker container ls --filter name=haproxy-service | /bin/grep haproxy-service"
    interval 5
    weight -20
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    interface ens192
    state BACKUP
    virtual_router_id 205
    priority 100
    virtual_ipaddress {
        10.64.137.205/32
    }
    track_script {
        haproxy_container
    }
}

blog20191008-18.sh

$ sudo docker service logs --tail 20 haproxy-service

blog20191008-19.sh

$ sudo docker service rm haproxy-service

blog20191008-20.sh

$ sudo docker service create \
  --mode global \
  --name haproxy-service \
  --network apache-network \
  --publish published=80,target=80,protocol=tcp,mode=host \
  --publish published=443,target=443,protocol=tcp,mode=host \
  --mount type=bind,src=/etc/haproxy/,dst=/etc/haproxy/,ro=true \
  --dns=127.0.0.11 \
  haproxytech/haproxy-debian:2.0

blog20191008-21.sh

$ sudo docker node update --label-add LB-NODE=yes dock2
$ sudo docker node update --label-add LB-NODE=yes dock3

blog20191008-22.sh

$ sudo docker service create \
  --mode global \
  --name haproxy-service \
  --network apache-network \
  --publish published=80,target=80,protocol=tcp,mode=host \
  --publish published=443,target=443,protocol=tcp,mode=host \
  --mount type=bind,src=/etc/haproxy/,dst=/etc/haproxy/,ro=false \
  --dns=127.0.0.11 \
  --constraint node.labels.LB-NODE==yes \
  haproxytech/haproxy-debian:2.0 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -L local_haproxy

blog20191008-23.sh

$ docker service update --constraint-add node.labels.LB-NODE==yes haproxy-service

blog20191008-24.sh

$ sudo docker service scale apache-Service=1

blog20191008-25.sh

$ sudo docker service scale apache-Service=6

blog20191008-26.sh

$ sudo docker kill --signal USR2 $(docker container ls --filter name=haproxy-service --quiet)

blog20191008-27.cfg

peers mypeers
    peer local_haproxy 127.0.0.1:1024

blog20191008-28.cfg

stick-table type ip size 1k expire 1h store http_req_cnt,http_req_rate(1m) peers mypeers 
http-request track-sc0 src

blog20191008-29.sh

$ sudo docker service create \
  --mode replicated \
  --replicas 1 \
  --name haproxy-service \
  --network apache-network \
  --publish published=80,target=80,protocol=tcp,mode=host \
  --publish published=443,target=443,protocol=tcp,mode=host \
  --mount type=bind,src=/etc/haproxy/,dst=/etc/haproxy/,ro=false \
  --dns=127.0.0.11 \
  haproxytech/haproxy-debian:2.0 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -L local_haproxy

blog20191008-full.cfg

global
    log     	 fd@2 local2
    chroot  	 /var/lib/haproxy
    pidfile 	 /var/run/haproxy.pid
    maxconn 	 4000
    user    	 haproxy
    group   	 haproxy
    stats socket /var/lib/haproxy/stats expose-fd listeners
    master-worker

resolvers docker
    nameserver dns1 127.0.0.11:53
    resolve_retries 3
    timeout resolve 1s
    timeout retry   1s
    hold other      10s
    hold refused    10s
    hold nx         10s
    hold timeout    10s
    hold valid      10s
    hold obsolete   10s

defaults
    timeout connect 10s
    timeout client 30s
    timeout server 30s
    log global
    mode http
    option httplog

frontend  fe_web
    bind *:80
    use_backend stat if { path -i /my-stats }
    default_backend be_apache_service 

backend be_apache_service
    balance roundrobin
    server-template apache- 6 apache-Service:80 check resolvers docker init-addr libc,none

backend be_apache_service_wrong_case
    balance roundrobin
    server-template apache- 6 apache-service:80 check resolvers docker init-addr libc,none

backend stat
    stats enable
    stats uri /my-stats
    stats refresh 15s
    stats show-legends
    stats show-node