Monitor our Server with Blackbox Exporter

README.md

Briefing

If you want to monitor your server with existing monitoring mainstream stack (Prometheus, Grafana) you can use Blackbox Exporter to monitor it.

Installation

wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.19.0/blackbox_exporter-0.19.0.linux-amd64.tar.gz
tar -xvf blackbox_exporter-0.19.0.linux-amd64.tar.gz
mv blackbox_exporter-0.19.0.linux-amd64/blackbox_exporter /usr/local/bin/
cd blackbox_exporter-0.19.0.linux-amd64/
sudo useradd -rs /bin/false blackbox_exporter
mkdir -p /etc/blackbox_exporter
chown -R blackbox_exporter:blackbox_exporter /etc/blackbox_exporter/

Blackbox Configuration

vim /etc/blackbox_exporter/blackbox.yml

---
modules:
  http_2xx:
    prober: http
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200]
      method: GET
      preferred_ip_protocol: ip4 ## default ip6
      fail_if_ssl: false
      fail_if_not_ssl: false
      tls_config:
        insecure_skip_verify: true
  http_post_2xx:
    prober: http
    http:
      method: POST
      preferred_ip_protocol: ip4
      fail_if_ssl: false
      fail_if_not_ssl: false
      tls_config:
        insecure_skip_verify: true
  tcp_connect:
    prober: tcp
    tcp:
      preferred_ip_protocol: ip4
  pop3s_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: ip4
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      preferred_ip_protocol: ip4
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
  icmp:
    prober: icmp
---

Create Blackbox Exporter Systemd Service

vim /etc/systemd/system/blackbox_exporter.service
---
[Unit]
Description=Blackbox Exporter
After=network.target

[Service]
User=blackbox_exporter
Group=blackbox_exporter
## add capability to use privileged raw ICMP sockets
CapabilityBoundingSet=CAP_NET_RAW
AmbientCapabilities=CAP_NET_RAW
Type=simple
ExecStart=/usr/local/bin/blackbox_exporter \
--config.file=/etc/blackbox_exporter/blackbox.yml

[Install]
WantedBy=multi-user.target
---

systemctl daemon-reload 
systemctl enable --now blackbox_exporter.service 
systemctl status blackbox_exporter.service

Add Blackbox Exporter to Prometheus Server

vim /etc/prometheus/prometheus.yml
---
  - job_name: 'blackbox_exporter_http'
    metrics_path: /probe
    params: 
      module: [http_2xx]
    static_configs:
            - targets:
              - https://target.url
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115

  - job_name: 'blackbox_exporter_icmp'
    metrics_path: /probe
    params: 
      module: [icmp]
    static_configs:
            - targets:
              - 1.1.1.1
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115
---

Visualize with Grafana

• Dashboard 7587

SSH Prometheus Target

vim /etc/prometheus/prometheus.yml
---
  - job_name: 'blackbox-ssh'
    metrics_path: /probe
    params:
      module: [ssh_banner]
    static_configs:
      - targets:
        - 172.15.15.152
    relabel_configs:
      # Ensure port is 22, pass as URL parameter
      - source_labels: [__address__]
        regex: (.*?)(:.*)?
        replacement: ${1}:22
        target_label: __param_target
      # Make instance label the target
      - source_labels: [__param_target]
        target_label: instance
      # Actually talk to the blackbox exporter though
      - target_label: __address__
        replacement: 127.0.0.1:9115
---

systemctl restart prometheus.service

DNS Monitoring

• Blackbox

  google.com:
    prober: dns
    timeout: 5s
    dns:
      transport_protocol: "udp"
      preferred_ip_protocol: "ip4"
      query_name: "google.com"
      query_type: "A"
      valid_rcodes:
        - NOERROR

• Prometheus

  - job_name: 'blackbox-dns-monitor'
    scrape_interval: 5s
    metrics_path: /probe
    relabel_configs:
    # Populate domain label with domain portion of __address__
    - source_labels: [__address__]
      regex: (.*):.*$
      replacement: $1
      target_label: domain
    # Populate instance label with dns server IP portion of __address__
    - source_labels: [__address__]
      regex: .*:(.*)$
      replacement: $1
      target_label: instance
    # Populate module URL parameter with domain portion of __address__
    # This is a parameter passed to the blackbox exporter
    - source_labels: [domain]
      target_label: __param_module
    # Populate target URL parameter with dns server IP
    - source_labels: [instance]
      target_label: __param_target
    # Populate __address__ with the address of the blackbox exporter to hit
    - target_label: __address__
      replacement: 172.16.1.2:9115
      
    static_configs:
      - targets:
        - google.com:8.8.8.8
        - google.com:1.1.1.1
        - google.com:9.9.9.9

• Grafana JSON

Grafana support on >= V8

Monitoring Stack Provisioning

• Grafana + Prometheus + NodeExporter Provisioning