Skip to content

Instantly share code, notes, and snippets.

@dougireton
Last active July 18, 2025 03:29
Show Gist options
  • Save dougireton/e90168985088d9dc131d042985a2ccd0 to your computer and use it in GitHub Desktop.
Save dougireton/e90168985088d9dc131d042985a2ccd0 to your computer and use it in GitHub Desktop.
Using Grafana Alloy to send data to Grafana Loki and Prometheus. Managed by Grafana Fleet Mgmt
## Use this with Grafana Fleet Mgmt
## See https://<yourorg>.grafana.net/a/grafana-collector-app/fleet-management/remote-configuration
prometheus.exporter.cadvisor "integrations_cadvisor" {
docker_only = true
}
discovery.relabel "integrations_cadvisor" {
targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets
rule {
target_label = "job"
replacement = "integrations/docker"
}
rule {
target_label = "instance"
replacement = constants.hostname
}
}
prometheus.relabel "integrations_cadvisor" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
source_labels = ["__name__"]
regex = "up|container_cpu_usage_seconds_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_limit_bytes|container_fs_usage_bytes|container_last_seen|container_memory_usage_bytes|container_network_receive_bytes_total|container_network_tcp_usage_total|container_network_transmit_bytes_total|container_spec_memory_reservation_limit_bytes|machine_memory_bytes|machine_scrape_error"
action = "keep"
}
# Create OTEL-compliant Grafana labels from running container docker label
# https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/
# https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/
rule {
source_labels = ["container_label_org_opencontainers_image_revision"]
target_label = "service_version"
}
rule {
source_labels = ["container_label_destination"]
target_label = "deployment_environment_name"
}
rule {
source_labels = ["container_label_service_name"]
target_label = "service_name"
}
}
prometheus.scrape "integrations_cadvisor" {
targets = discovery.relabel.integrations_cadvisor.output
forward_to = [prometheus.relabel.integrations_cadvisor.receiver]
}
prometheus.remote_write "metrics_service" {
endpoint {
url = "https://prometheus-prod-xxx.grafana.net/api/prom/push"
basic_auth {
username = "1234567" # Your Grafana Cloud Prometheus username.
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
discovery.docker "logs_integrations_docker" {
host = "unix:///var/run/docker.sock"
refresh_interval = "5s"
}
discovery.relabel "logs_integrations_docker" {
targets = []
rule {
target_label = "job"
replacement = "integrations/docker"
}
rule {
target_label = "instance"
replacement = constants.hostname
}
rule {
source_labels = ["__meta_docker_container_name"]
regex = "/(.*)"
target_label = "container"
}
rule {
source_labels = ["__meta_docker_container_log_stream"]
target_label = "stream"
}
# Drop the kamal-proxy container logs as they are not useful for monitoring
# and can cause excessive log volume.
rule {
source_labels = ["__meta_docker_container_name"]
regex = "/kamal-proxy"
action = "drop"
}
# Create OTEL-compliant Grafana labels from running container docker label
# https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/
# https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/
rule {
source_labels = ["__meta_docker_container_label_org_opencontainers_image_revision"]
target_label = "service_version"
}
rule {
source_labels = ["__meta_docker_container_label_destination"]
target_label = "deployment_environment_name"
}
rule {
source_labels = ["__meta_docker_container_label_service_name"]
target_label = "service_name"
}
}
loki.source.docker "logs_integrations_docker" {
host = "unix:///var/run/docker.sock"
targets = discovery.docker.logs_integrations_docker.targets
forward_to = [loki.write.grafana_cloud_loki.receiver]
relabel_rules = discovery.relabel.logs_integrations_docker.rules
refresh_interval = "5s"
}
loki.write "grafana_cloud_loki" {
endpoint {
url = "https://logs-prod-nnn.grafana.net/loki/api/v1/push"
basic_auth {
username = "1234567" # Your Grafana Cloud Loki username.
password = sys.env("GCLOUD_RW_API_KEY")
}
}
}
#cloud-config
package_update: true
package_upgrade: true
packages:
- alloy # Grafana Alloy
- ca-certificates
- docker.io
- gnupg
apt:
sources:
grafana:
source: "deb https://apt.grafana.com stable main"
key: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQGNBGTnhmkBDADUE+SzjRRyitIm1siGxiHlIlnn6KO4C4GfEuV+PNzqxvwYO+1r
mcKlGDU0ugo8ohXruAOC77Kwc4keVGNU89BeHvrYbIftz/yxEneuPsCbGnbDMIyC
k44UOetRtV9/59Gj5YjNqnsZCr+e5D/JfrHUJTTwKLv88A9eHKxskrlZr7Un7j3i
Ef3NChlOh2Zk9Wfk8IhAqMMTferU4iTIhQk+5fanShtXIuzBaxU3lkzFSG7VuAH4
CBLPWitKRMn5oqXUE0FZbRYL/6Qz0Gt6YCJsZbaQ3Am7FCwWCp9+ZHbR9yU+bkK0
Dts4PNx4Wr9CktHIvbypT4Lk2oJEPWjcCJQHqpPQZXbnclXRlK5Ea0NVpaQdGK+v
JS4HGxFFjSkvTKAZYgwOk93qlpFeDML3TuSgWxuw4NIDitvewudnaWzfl9tDIoVS
Bb16nwJ8bMDzovC/RBE14rRKYtMLmBsRzGYHWd0NnX+FitAS9uURHuFxghv9GFPh
eTaXvc4glM94HBUAEQEAAbQmR3JhZmFuYSBMYWJzIDxlbmdpbmVlcmluZ0BncmFm
YW5hLmNvbT6JAdQEEwEKAD4WIQS1Oud7rbYwpoMEYAWWP6J3EEWFRQUCZOeGaQIb
AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCWP6J3EEWFRUiADACa
i+xytv2keEFJWjXNnFAx6/obnHRcXOI3w6nH/zL8gNI7YN5jcdQT2NYvKVYTb3fW
GuMsjHWgat5Gq3AtJrOKABpZ6qeYNPk0Axn/dKtOTwXjZ4pKX3bbUYvVfs0fCEZv
B0HHIj2wI9kgMpoTrkj22LE8layZTPOoQ+3/FbLzS8hN3CYZj25mHN7bpZq8EbV3
8FW9EU0HM0tg6CvoxkRiVqAuAC0KnVIZAdhD4dlYKuncq64nMvT1A5wxSYbnE+uf
mnWQQhhS6BOwRqN054yw1FrWNDFsvnOSHmr8dIiriv+aZYvx5JQFJ7oZP3LwdYyg
ocQcAJA8HFTIk3P6uJiIF/zdDzocgdKs+IYDoId0hxX7sGCvqdrsveq8n3m7uQiN
7FvSiV0eXIdV4F7340kc8EKiYwpuYSaZX0UWKLenzlUvD+W4pZCWtoXzPsW7PKUt
q1xdW0+NY+AGLCvSJCc5F4S5kFCObfBAYBbldjwwJFocdq/YOvvWYTPyV7kJeJS5
AY0EZOeGaQEMALNIFUricEIwtZiX7vSDjwxobbqPKqzdek8x3ud0CyYlrbGHy0k+
FDEXstjJQQ1s9rjJSu3sv5wyg9GDAUH3nzO976n/ZZvKPti3p2XU2UFx5gYkaaFV
D56yYxqGY0YU5ft6BG+RUz3iEPg3UBUzt0sCIYnG9+CsDqGOnRYIIa46fu2/H9Vu
8JvvSq9xbsK9CfoQDkIcoQOixPuI4P7eHtswCeYR/1LUTWEnYQWsBCf57cEpzR6t
7mlQnzQo9z4i/kp4S0ybDB77wnn+isMADOS+/VpXO+M7Zj5tpfJ6PkKch3SGXdUy
3zht8luFOYpJr2lVzp7n3NwB4zW08RptTzTgFAaW/NH2JjYI+rDvQm4jNs08Dtsp
nm4OQvBA9Df/6qwMEOZ9i10ixqk+55UpQFJ3nf4uKlSUM7bKXXVcD/odq804Y/K4
y3csE059YVIyaPexEvYSYlHE2odJWRg2Q1VehmrOSC8Qps3xpU7dTHXD74ZpaYbr
haViRS5v/lCsiwARAQABiQG8BBgBCgAmFiEEtTrne622MKaDBGAFlj+idxBFhUUF
AmTnhmkCGwwFCQPCZwAACgkQlj+idxBFhUUNbQv8DCcfi3GbWfvp9pfY0EJuoFJX
LNgci7z7smXq7aqDp2huYQ+MulnPAydjRCVW2fkHItF2Ks6l+2/8t5Xz0eesGxST
xTyR31ARENMXaq78Lq+itZ+usOSDNuwJcEmJM6CceNMLs4uFkX2GRYhchkry7P0C
lkLxUTiB43ooi+CqILtlNxH7kM1O4Ncs6UGZMXf2IiG9s3JDCsYVPkC5QDMOPkTy
2ZriF56uPerlJveF0dC61RZ6RlM3iSJ9Fwvea0Oy4rwkCcs5SHuwoDTFyxiyz0QC
9iqi3fG3iSbLvY9UtJ6X+BtDqdXLAT9Pq527mukPP3LwpEqFVyNQKnGLdLOu2YXc
TWWWseSQkHRzBmjD18KTD74mg4aXxEabyT4snrXpi5+UGLT4KXGV5syQO6Lc0OGw
9O/0qAIU+YW7ojbKv8fr+NB31TGhGYWASjYlN1NvPotRAK6339O0/Rqr9xGgy3AY
SR+ic2Y610IM7xccKuTVAW9UofKQwJZChqae9VVZ
=J9CI
-----END PGP PUBLIC KEY BLOCK-----
write_files:
- path: /etc/alloy/config.alloy
owner: root:root
permissions: '0644'
content: |
remotecfg {
url = "<alloyConfig.remoteCfg.url>"
basic_auth {
username = "<alloyConfig.remoteCfg.basicAuth.username>"
password = "<grafanaRWApiKey>"
}
id = constants.hostname
attributes = "docker-host=true, app=my-app" # attributes for Alloy. I use "docker-host: true" to indicate that this is a Docker host to apply Docker-specific Fleet Mgmt configuration.
poll_frequency = "5m"
}
- path: /etc/systemd/system/alloy.service.d/10-alloy.conf
owner: root:root
permissions: '0644'
content: |
[Service]
User=root
Environment=GCLOUD_RW_API_KEY="<grafanaRWApiKey>"
Environment=GCLOUD_FM_COLLECTOR_ID=%H
runcmd:
- [ usermod, --append, --groups, docker, ubuntu ]
- [ usermod, --append, --groups, docker, alloy ]
- [ systemctl, daemon-reload ]
- [ systemctl, enable, --now, alloy.service ]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment