Docker Container Logging using Promtail

promtail_docker_logs.md

Inspired By: grafana/loki#333

• docker-compose.yml

version: "3"

networks:
  loki:

services:
  loki:
    image: grafana/loki:1.4.1
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - loki

  promtail:
    image: grafana/promtail:1.4.1
    volumes:
      - /var/lib/docker/containers:/var/lib/docker/containers
      - /home/ubuntu/docker-config.yml:/etc/promtail/docker-config.yml
    command: -config.file=/etc/promtail/docker-config.yml
    networks:
      - loki

  grafana:
    image: grafana/grafana:master
    ports:
      - "3000:3000"
    networks:
      - loki

• docker-config.yml

server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:

- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: varlogs
      __path__: /var/log/*log

- job_name: containers
  entry_parser: raw

  static_configs:
  - targets:
      - localhost
    labels:
      job: containerlogs
      __path__: /var/lib/docker/containers/*/*log

  # --log-opt tag="{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
  pipeline_stages:

  - json:
      expressions:
        stream: stream
        attrs: attrs
        tag: attrs.tag

  - regex:
      expression: (?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))
      source: "tag"

  - labels:
      tag:
      stream:
      image_name:
      container_name:
      image_id:
      container_id:

$ docker-compose up -d

$ docker run -itd --name nginxapp  -p 8080:80 --log-driver json-file --log-opt tag="{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" nginx
$ curl http://localhost:8080/?foo=bar

Screenshot:

Setup with less labels:

• docker-config.yml

server:
  http_listen_address: 0.0.0.0
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: containers
  entry_parser: raw

  static_configs:
  - targets:
      - localhost
    labels:
      job: containerlogs
      cluster: multipass-cluster
      __path__: /var/lib/docker/containers/*/*log

  # --log-opt tag="{{.Name}}"
  pipeline_stages:

  - json:
      expressions:
        stream: stream
        attrs: attrs
        tag: attrs.tag

  - regex:
      expression: (?P<container_name>(?:[^|]*[^|]))
      source: "tag"

  - labels:
      #tag:
      stream:
      container_name:

$ docker run -itd --name nginxapp3  -p 8080:80 --log-driver json-file --log-opt tag="{{.Name}}" nginx
$ curl -XGET -A "Mozilla" --refer http://bot.com/scrape.html http://localhost:8080/?foo=barx

Screenshot:

entry_parser: raw
it does not work

@chrisbecke you can remove entry_parser: raw if you want to use new versions of promtail
@Pl8tinium thank you very much, best solution for me!!!

@Pl8tinium dude you saved my life, no joke! thanks. I was dealing with the docker plugin but it locked all my containers, thanks to promtail and your answer i could implement the solution.

The docker plugin has a hard dependency to a loki instance. So if the Loki instance is down, you aren't able to start your containers, which is an absolute nogo. I would rather miss the log messages than miss my running containers...

@Pl8tinium Thank you so much！Great solution！

@bykof I cant't agree more. Fatal Lesson! It's strongly not recommended to use loki plugin in docker ，especially in prod env.