Last active
July 18, 2025 03:29
-
-
Save dougireton/e90168985088d9dc131d042985a2ccd0 to your computer and use it in GitHub Desktop.
Using Grafana Alloy to send data to Grafana Loki and Prometheus. Managed by Grafana Fleet Mgmt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
## Use this with Grafana Fleet Mgmt | |
## See https://<yourorg>.grafana.net/a/grafana-collector-app/fleet-management/remote-configuration | |
prometheus.exporter.cadvisor "integrations_cadvisor" { | |
docker_only = true | |
} | |
discovery.relabel "integrations_cadvisor" { | |
targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets | |
rule { | |
target_label = "job" | |
replacement = "integrations/docker" | |
} | |
rule { | |
target_label = "instance" | |
replacement = constants.hostname | |
} | |
} | |
prometheus.relabel "integrations_cadvisor" { | |
forward_to = [prometheus.remote_write.metrics_service.receiver] | |
rule { | |
source_labels = ["__name__"] | |
regex = "up|container_cpu_usage_seconds_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_limit_bytes|container_fs_usage_bytes|container_last_seen|container_memory_usage_bytes|container_network_receive_bytes_total|container_network_tcp_usage_total|container_network_transmit_bytes_total|container_spec_memory_reservation_limit_bytes|machine_memory_bytes|machine_scrape_error" | |
action = "keep" | |
} | |
# Create OTEL-compliant Grafana labels from running container docker label | |
# https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/ | |
# https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ | |
rule { | |
source_labels = ["container_label_org_opencontainers_image_revision"] | |
target_label = "service_version" | |
} | |
rule { | |
source_labels = ["container_label_destination"] | |
target_label = "deployment_environment_name" | |
} | |
rule { | |
source_labels = ["container_label_service_name"] | |
target_label = "service_name" | |
} | |
} | |
prometheus.scrape "integrations_cadvisor" { | |
targets = discovery.relabel.integrations_cadvisor.output | |
forward_to = [prometheus.relabel.integrations_cadvisor.receiver] | |
} | |
prometheus.remote_write "metrics_service" { | |
endpoint { | |
url = "https://prometheus-prod-xxx.grafana.net/api/prom/push" | |
basic_auth { | |
username = "1234567" # Your Grafana Cloud Prometheus username. | |
password = sys.env("GCLOUD_RW_API_KEY") | |
} | |
} | |
} | |
discovery.docker "logs_integrations_docker" { | |
host = "unix:///var/run/docker.sock" | |
refresh_interval = "5s" | |
} | |
discovery.relabel "logs_integrations_docker" { | |
targets = [] | |
rule { | |
target_label = "job" | |
replacement = "integrations/docker" | |
} | |
rule { | |
target_label = "instance" | |
replacement = constants.hostname | |
} | |
rule { | |
source_labels = ["__meta_docker_container_name"] | |
regex = "/(.*)" | |
target_label = "container" | |
} | |
rule { | |
source_labels = ["__meta_docker_container_log_stream"] | |
target_label = "stream" | |
} | |
# Drop the kamal-proxy container logs as they are not useful for monitoring | |
# and can cause excessive log volume. | |
rule { | |
source_labels = ["__meta_docker_container_name"] | |
regex = "/kamal-proxy" | |
action = "drop" | |
} | |
# Create OTEL-compliant Grafana labels from running container docker label | |
# https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/ | |
# https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ | |
rule { | |
source_labels = ["__meta_docker_container_label_org_opencontainers_image_revision"] | |
target_label = "service_version" | |
} | |
rule { | |
source_labels = ["__meta_docker_container_label_destination"] | |
target_label = "deployment_environment_name" | |
} | |
rule { | |
source_labels = ["__meta_docker_container_label_service_name"] | |
target_label = "service_name" | |
} | |
} | |
loki.source.docker "logs_integrations_docker" { | |
host = "unix:///var/run/docker.sock" | |
targets = discovery.docker.logs_integrations_docker.targets | |
forward_to = [loki.write.grafana_cloud_loki.receiver] | |
relabel_rules = discovery.relabel.logs_integrations_docker.rules | |
refresh_interval = "5s" | |
} | |
loki.write "grafana_cloud_loki" { | |
endpoint { | |
url = "https://logs-prod-nnn.grafana.net/loki/api/v1/push" | |
basic_auth { | |
username = "1234567" # Your Grafana Cloud Loki username. | |
password = sys.env("GCLOUD_RW_API_KEY") | |
} | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#cloud-config | |
package_update: true | |
package_upgrade: true | |
packages: | |
- alloy # Grafana Alloy | |
- ca-certificates | |
- docker.io | |
- gnupg | |
apt: | |
sources: | |
grafana: | |
source: "deb https://apt.grafana.com stable main" | |
key: | | |
-----BEGIN PGP PUBLIC KEY BLOCK----- | |
mQGNBGTnhmkBDADUE+SzjRRyitIm1siGxiHlIlnn6KO4C4GfEuV+PNzqxvwYO+1r | |
mcKlGDU0ugo8ohXruAOC77Kwc4keVGNU89BeHvrYbIftz/yxEneuPsCbGnbDMIyC | |
k44UOetRtV9/59Gj5YjNqnsZCr+e5D/JfrHUJTTwKLv88A9eHKxskrlZr7Un7j3i | |
Ef3NChlOh2Zk9Wfk8IhAqMMTferU4iTIhQk+5fanShtXIuzBaxU3lkzFSG7VuAH4 | |
CBLPWitKRMn5oqXUE0FZbRYL/6Qz0Gt6YCJsZbaQ3Am7FCwWCp9+ZHbR9yU+bkK0 | |
Dts4PNx4Wr9CktHIvbypT4Lk2oJEPWjcCJQHqpPQZXbnclXRlK5Ea0NVpaQdGK+v | |
JS4HGxFFjSkvTKAZYgwOk93qlpFeDML3TuSgWxuw4NIDitvewudnaWzfl9tDIoVS | |
Bb16nwJ8bMDzovC/RBE14rRKYtMLmBsRzGYHWd0NnX+FitAS9uURHuFxghv9GFPh | |
eTaXvc4glM94HBUAEQEAAbQmR3JhZmFuYSBMYWJzIDxlbmdpbmVlcmluZ0BncmFm | |
YW5hLmNvbT6JAdQEEwEKAD4WIQS1Oud7rbYwpoMEYAWWP6J3EEWFRQUCZOeGaQIb | |
AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCWP6J3EEWFRUiADACa | |
i+xytv2keEFJWjXNnFAx6/obnHRcXOI3w6nH/zL8gNI7YN5jcdQT2NYvKVYTb3fW | |
GuMsjHWgat5Gq3AtJrOKABpZ6qeYNPk0Axn/dKtOTwXjZ4pKX3bbUYvVfs0fCEZv | |
B0HHIj2wI9kgMpoTrkj22LE8layZTPOoQ+3/FbLzS8hN3CYZj25mHN7bpZq8EbV3 | |
8FW9EU0HM0tg6CvoxkRiVqAuAC0KnVIZAdhD4dlYKuncq64nMvT1A5wxSYbnE+uf | |
mnWQQhhS6BOwRqN054yw1FrWNDFsvnOSHmr8dIiriv+aZYvx5JQFJ7oZP3LwdYyg | |
ocQcAJA8HFTIk3P6uJiIF/zdDzocgdKs+IYDoId0hxX7sGCvqdrsveq8n3m7uQiN | |
7FvSiV0eXIdV4F7340kc8EKiYwpuYSaZX0UWKLenzlUvD+W4pZCWtoXzPsW7PKUt | |
q1xdW0+NY+AGLCvSJCc5F4S5kFCObfBAYBbldjwwJFocdq/YOvvWYTPyV7kJeJS5 | |
AY0EZOeGaQEMALNIFUricEIwtZiX7vSDjwxobbqPKqzdek8x3ud0CyYlrbGHy0k+ | |
FDEXstjJQQ1s9rjJSu3sv5wyg9GDAUH3nzO976n/ZZvKPti3p2XU2UFx5gYkaaFV | |
D56yYxqGY0YU5ft6BG+RUz3iEPg3UBUzt0sCIYnG9+CsDqGOnRYIIa46fu2/H9Vu | |
8JvvSq9xbsK9CfoQDkIcoQOixPuI4P7eHtswCeYR/1LUTWEnYQWsBCf57cEpzR6t | |
7mlQnzQo9z4i/kp4S0ybDB77wnn+isMADOS+/VpXO+M7Zj5tpfJ6PkKch3SGXdUy | |
3zht8luFOYpJr2lVzp7n3NwB4zW08RptTzTgFAaW/NH2JjYI+rDvQm4jNs08Dtsp | |
nm4OQvBA9Df/6qwMEOZ9i10ixqk+55UpQFJ3nf4uKlSUM7bKXXVcD/odq804Y/K4 | |
y3csE059YVIyaPexEvYSYlHE2odJWRg2Q1VehmrOSC8Qps3xpU7dTHXD74ZpaYbr | |
haViRS5v/lCsiwARAQABiQG8BBgBCgAmFiEEtTrne622MKaDBGAFlj+idxBFhUUF | |
AmTnhmkCGwwFCQPCZwAACgkQlj+idxBFhUUNbQv8DCcfi3GbWfvp9pfY0EJuoFJX | |
LNgci7z7smXq7aqDp2huYQ+MulnPAydjRCVW2fkHItF2Ks6l+2/8t5Xz0eesGxST | |
xTyR31ARENMXaq78Lq+itZ+usOSDNuwJcEmJM6CceNMLs4uFkX2GRYhchkry7P0C | |
lkLxUTiB43ooi+CqILtlNxH7kM1O4Ncs6UGZMXf2IiG9s3JDCsYVPkC5QDMOPkTy | |
2ZriF56uPerlJveF0dC61RZ6RlM3iSJ9Fwvea0Oy4rwkCcs5SHuwoDTFyxiyz0QC | |
9iqi3fG3iSbLvY9UtJ6X+BtDqdXLAT9Pq527mukPP3LwpEqFVyNQKnGLdLOu2YXc | |
TWWWseSQkHRzBmjD18KTD74mg4aXxEabyT4snrXpi5+UGLT4KXGV5syQO6Lc0OGw | |
9O/0qAIU+YW7ojbKv8fr+NB31TGhGYWASjYlN1NvPotRAK6339O0/Rqr9xGgy3AY | |
SR+ic2Y610IM7xccKuTVAW9UofKQwJZChqae9VVZ | |
=J9CI | |
-----END PGP PUBLIC KEY BLOCK----- | |
write_files: | |
- path: /etc/alloy/config.alloy | |
owner: root:root | |
permissions: '0644' | |
content: | | |
remotecfg { | |
url = "<alloyConfig.remoteCfg.url>" | |
basic_auth { | |
username = "<alloyConfig.remoteCfg.basicAuth.username>" | |
password = "<grafanaRWApiKey>" | |
} | |
id = constants.hostname | |
attributes = "docker-host=true, app=my-app" # attributes for Alloy. I use "docker-host: true" to indicate that this is a Docker host to apply Docker-specific Fleet Mgmt configuration. | |
poll_frequency = "5m" | |
} | |
- path: /etc/systemd/system/alloy.service.d/10-alloy.conf | |
owner: root:root | |
permissions: '0644' | |
content: | | |
[Service] | |
User=root | |
Environment=GCLOUD_RW_API_KEY="<grafanaRWApiKey>" | |
Environment=GCLOUD_FM_COLLECTOR_ID=%H | |
runcmd: | |
- [ usermod, --append, --groups, docker, ubuntu ] | |
- [ usermod, --append, --groups, docker, alloy ] | |
- [ systemctl, daemon-reload ] | |
- [ systemctl, enable, --now, alloy.service ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment