Set up Datadog agent and cloud-init-logger

dd_labels.yaml

labels:
  com.datadoghq.ad.checks: '{"docker": {"instances": [{"host": "%%host%%", "port": 8081}]}}'
  com.datadoghq.ad.tags: '["org:${ORG_NAME}", "project:${WORKFLOW_NAME}", "worker_type:${WORKER_DOCKER_SUBDIR}", "log_parser:${LOG_PARSER}"]'
  com.datadoghq.ad.logs: >-
    [{
      "type": "docker",
      "source": "${ORG_NAME}_${WORKFLOW_NAME}", 
      "service": "${WORKER_DOCKER_SUBDIR}_${APPLICATION}", 
      "log_processing_rules":[{
        "type": "multi_line",
        "name": "log_start_with_date",
        "pattern": "\\[\\d{4}-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01])"
      }]
    }]

docker-compose.cloud-init-logger.yaml

version: '3'
services:
  cloud-init-logger:
    image: debian:latest
    container_name: cloud-init-logger
    labels:
      com.datadoghq.ad.checks: '{"docker": {"instances": [{"host": "%%host%%", "port": 8081}]}}'
      com.datadoghq.ad.logs: |
        [{"type":"docker", "source": "docker", "service": "cloud-init-logger"}]
    volumes:
      - /var/log/cloud-init-output.log:/var/log/cloud-init-output.log:ro
    command:
      - bash
      - -c
      - |
        tail -n +1 -f /var/log/cloud-init-output.log;

docker-compose.dd-agent.yaml

# envsubst < docker-compose.dd-agent.yaml | sponge docker-compose.dd-agent.yaml
version: '3'
services:
  datadog:
    image: gcr.io/datadoghq/agent:7
    container_name: dd-agent
    pid: host
    environment:
     - DD_API_KEY=${DD_API_KEY}
     - DD_SITE=datadoghq.com
     - DD_PROCESS_AGENT_ENABLED=true
     - DD_CONTAINER_EXCLUDE=name:dd-agent
     - DD_LOGS_ENABLED=true
     - DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true
     - DD_LOG_LEVEL=debug
     - DD_TAGS="org:${ORG_NAME} project:${WORKFLOW_NAME} worker-type:${WORKER_DOCKER_SUBDIR}"
    volumes:
     - /var/run/docker.sock:/var/run/docker.sock
     - /var/lib/docker/containers:/var/lib/docker/containers:ro
     - /proc/:/host/proc/:ro
     - /opt/datadog-agent/run:/opt/datadog-agent/run:rw
     - /sys/fs/cgroup:/host/sys/fs/cgroup:ro

inject_dd_labels.sh

#!/bin/bash
export ORG_NAME=$1
export WORKFLOW_NAME=$2
export WORKER_DOCKER_SUBDIR=$3
export APPLICATION=$4
export LOG_PARSER=$5

export DD_LABEL_FILE=$6
export INJECTED_FILE=$7

envsubst < ${DD_LABEL_FILE} | sponge ${DD_LABEL_FILE}
yq eval-all \
  'select(fileIndex==0).services.[env(WORKER_DOCKER_SUBDIR)].labels."com.datadoghq.ad.checks" = select(fileIndex==1).labels."com.datadoghq.ad.checks" | select(fileIndex==0).services.[env(WORKER_DOCKER_SUBDIR)].labels."com.datadoghq.ad.tags" = select(fileIndex==1).labels."com.datadoghq.ad.tags" | select(fileIndex==0).services.[env(WORKER_DOCKER_SUBDIR)].labels."com.datadoghq.ad.logs" = select(fileIndex==1).labels."com.datadoghq.ad.logs" | select(fileIndex==0)' \
   ${INJECTED_FILE} ${DD_LABEL_FILE} | sponge ${INJECTED_FILE}

setup_dd_agent.sh

#!/bin/bash
export DD_API_KEY=$1
export ORG_NAME=$2
export WORKFLOW_NAME=$3
export WORKER_DOCKER_SUBDIR=$4
export SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
cd $SCRIPT_DIR
envsubst < docker-compose.dd-agent.yaml | sponge docker-compose.dd-agent.yaml
docker compose -f docker-compose.dd-agent.yaml -f docker-compose.cloud-init-logger.yaml up -d
# docker compose -f docker-compose.dd-agent.yaml up -d