Set up AWS CloudWatch Agent on an EC2 instance

amazon-cloudwatch-agent-cpu.json

{
    "agent": {
        "metrics_collection_interval": 60,
        "run_as_user": "root"
    },
    "metrics": {
        "namespace": "$CONTRACT",
        "aggregation_dimensions": [["InstanceType"]],
        "append_dimensions": {
            "InstanceId": "${aws:InstanceId}",
            "InstanceType": "${aws:InstanceType}"
        },
        "metrics_collected": {
            "disk": {
                "measurement": ["used_percent"],
                "metrics_collection_interval": 60,
                "resources": ["/"],
                "append_dimensions": {
                    "Scope": "$SCOPE",
                    "Contract": "$CONTRACT"
                }
            },
            "mem": {
                "measurement": ["mem_used_percent"],
                "metrics_collection_interval": 60,
                "append_dimensions": {
                    "Scope": "$SCOPE",
                    "Contract": "$CONTRACT"
                }
            }
        }
    }
}

amazon-cloudwatch-agent-gpu.json

{
    "agent": {
        "metrics_collection_interval": 60,
        "run_as_user": "root"
    },
    "metrics": {
        "namespace": "$CONTRACT",
        "aggregation_dimensions": [["InstanceType"]],
        "append_dimensions": {
            "InstanceId": "${aws:InstanceId}",
            "InstanceType": "${aws:InstanceType}"
        },
        "metrics_collected": {
            "disk": {
                "measurement": ["used_percent"],
                "metrics_collection_interval": 60,
                "resources": ["/"],
                "append_dimensions": {
                    "Scope": "$SCOPE",
                    "Contract": "$CONTRACT"
                }
            },
            "mem": {
                "measurement": ["mem_used_percent"],
                "metrics_collection_interval": 60,
                "append_dimensions": {
                    "Scope": "$SCOPE",
                    "Contract": "$CONTRACT"
                }
            },
            "nvidia_gpu": {
                "measurement": ["utilization_gpu"],
                "metrics_collection_interval": 60,
                "append_dimensions": {
                    "Scope": "$SCOPE",
                    "Contract": "$CONTRACT"
                }
            }
        }
    }
}

cloudwatch_agent.md

Install cloud watch agent

wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -P /tmp
dpkg -i -E /tmp/amazon-cloudwatch-agent.deb

Write cloud watch agent json config file

Documentation recommends to save the .json to /opt/aws/amazon-cloudwatch-agent/etc, but the .json will be auto translate to .toml file. So save the .json to /opt/aws/amazon-cloudwatch-agent/etc-json instead

{
    "agent": {
        "metrics_collection_interval": 60,
        "run_as_user": "root"
    },
    "metrics": {
        "namespace": "",
        "aggregation_dimensions": [["InstanceType"]],
        "append_dimensions": {
            "InstanceId": "${aws:InstanceId}",
            "InstanceType": "${aws:InstanceType}"
        },
        "metrics_collected": {
            "disk": {
                "measurement": ["used_percent"],
                "metrics_collection_interval": 60,
                "resources": ["/"],
                "append_dimensions": {
                    "Scope": "",
                    "Contract": ""
                }
            },
            "mem": {
                "measurement": ["mem_used_percent"],
                "metrics_collection_interval": 60,
                "append_dimensions": {
                    "Scope": "",
                    "Contract": ""
                }
            }
        }
    }
}

Create/Use existing role

• Add policy CloudWatchAgentServerPolicy
• Assign role to the EC2 instance

Start cloud watch agent

## Run every time after updating the config file
sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc-json/amazon-cloudwatch-agent.json

setup_cloudwatch_agent.sh

#!/bin/bash
wget https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb -P /tmp
dpkg -i -E /tmp/amazon-cloudwatch-agent.deb

export SCOPE=$1
export CONTRACT=$2
export GPU_CPU=$3
mkdir /opt/aws/amazon-cloudwatch-agent/etc-json
cat $( dirname -- "$0"; )/amazon-cloudwatch-agent-${GPU_CPU}.json | envsubst > \
/opt/aws/amazon-cloudwatch-agent/etc-json/amazon-cloudwatch-agent.json
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c \
file:/opt/aws/amazon-cloudwatch-agent/etc-json/amazon-cloudwatch-agent.json