Last active
February 5, 2022 02:59
-
-
Save kesor/c681ed58f2211eb0c61c4328f01ae940 to your computer and use it in GitHub Desktop.
An ElasticSearch Index template for CloudTrail events
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT _template/cloudtrail | |
| { | |
| "index_patterns": ["cloudtrail-*"], | |
| "settings": { | |
| "number_of_shards": 1, | |
| "mapping": { | |
| "total_fields": { | |
| "limit": 10000 | |
| } | |
| } | |
| }, | |
| "mappings": { | |
| "doc": { | |
| "properties": { | |
| "@timestamp" : { | |
| "type" : "date" | |
| }, | |
| "@version" : { | |
| "type" : "integer" | |
| }, | |
| "additionalEventData" : { | |
| "properties" : { | |
| "LoginTo" : { | |
| "type" : "keyword" | |
| }, | |
| "MFAUsed" : { | |
| "type" : "keyword" | |
| }, | |
| "MobileVersion" : { | |
| "type" : "keyword" | |
| }, | |
| "Note" : { | |
| "type" : "text" | |
| }, | |
| "SamlProviderArn" : { | |
| "type" : "keyword" | |
| }, | |
| "vpcEndpointId" : { | |
| "type" : "keyword" | |
| } | |
| } | |
| }, | |
| "apiVersion" : { | |
| "type" : "keyword" | |
| }, | |
| "awsRegion" : { | |
| "type" : "keyword" | |
| }, | |
| "errorCode" : { | |
| "type" : "keyword" | |
| }, | |
| "errorMessage" : { | |
| "type" : "text" | |
| }, | |
| "eventID" : { | |
| "type" : "keyword" | |
| }, | |
| "eventName" : { | |
| "type" : "keyword" | |
| }, | |
| "eventSource" : { | |
| "type" : "keyword" | |
| }, | |
| "eventType" : { | |
| "type" : "keyword" | |
| }, | |
| "eventVersion" : { | |
| "type" : "double" | |
| }, | |
| "readOnly" : { | |
| "type" : "boolean" | |
| }, | |
| "recipientAccountId" : { | |
| "type" : "keyword" | |
| }, | |
| "requestID" : { | |
| "type" : "keyword" | |
| }, | |
| "requestParameters" : { | |
| "type": "object", | |
| "properties": { | |
| "iamInstanceProfile": { | |
| "type": "object", | |
| "properties": { | |
| "name": { | |
| "type": "keyword" | |
| }, | |
| "arn": { | |
| "type": "keyword" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "resources" : { | |
| "properties" : { | |
| "ARN" : { | |
| "type" : "keyword" | |
| }, | |
| "accountId" : { | |
| "type" : "keyword" | |
| }, | |
| "type" : { | |
| "type" : "keyword" | |
| } | |
| } | |
| }, | |
| "responseElements" : { | |
| "type": "object" | |
| }, | |
| "serviceEventDetails" : { | |
| "type": "object" | |
| }, | |
| "sharedEventID" : { | |
| "type" : "keyword" | |
| }, | |
| "sourceIPAddress" : { | |
| "type" : "keyword" | |
| }, | |
| "tags" : { | |
| "type" : "keyword" | |
| }, | |
| "type" : { | |
| "type" : "keyword" | |
| }, | |
| "userAgent" : { | |
| "type" : "keyword" | |
| }, | |
| "userIdentity" : { | |
| "properties" : { | |
| "accessKeyId" : { | |
| "type" : "keyword" | |
| }, | |
| "accountId" : { | |
| "type" : "keyword" | |
| }, | |
| "arn" : { | |
| "type" : "keyword" | |
| }, | |
| "identityProvider" : { | |
| "type" : "keyword" | |
| }, | |
| "invokedBy" : { | |
| "type" : "keyword" | |
| }, | |
| "principalId" : { | |
| "type" : "keyword" | |
| }, | |
| "sessionContext" : { | |
| "properties" : { | |
| "attributes" : { | |
| "properties" : { | |
| "creationDate" : { | |
| "type" : "date" | |
| }, | |
| "mfaAuthenticated" : { | |
| "type" : "boolean" | |
| } | |
| } | |
| }, | |
| "sessionIssuer" : { | |
| "properties" : { | |
| "accountId" : { | |
| "type" : "keyword" | |
| }, | |
| "arn" : { | |
| "type" : "keyword" | |
| }, | |
| "principalId" : { | |
| "type" : "keyword" | |
| }, | |
| "type" : { | |
| "type" : "keyword" | |
| }, | |
| "userName" : { | |
| "type" : "keyword" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "type" : { | |
| "type" : "keyword" | |
| }, | |
| "userName" : { | |
| "type" : "keyword" | |
| } | |
| } | |
| }, | |
| "vpcEndpointId" : { | |
| "type" : "keyword" | |
| } | |
| } | |
| } | |
| } | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| input { | |
| s3 { | |
| bucket => "our-happy-cloudtrail-bucket" | |
| type => "cloudtrail" | |
| codec => "cloudtrail" | |
| # the s3-input plugins lists ALL objects in this prefix each time | |
| # it checks if something was added ... very expensive for big buckets | |
| prefix => "CT/AWSLogs/123456789012/CloudTrail/us-east-1/2019/05/" | |
| exclude_pattern => "/(2018|2019/01|2019/02|2019/03|2019/04)/" | |
| watch_for_new_files => true | |
| } | |
| } | |
| filter { | |
| metrics { | |
| meter => [ "events" ] | |
| add_tag => "metric" | |
| } | |
| if [type] == "cloudtrail" { | |
| if [requestParameters][iamInstanceProfile] { | |
| # coerce requestParameters.iamInstanceProfile | |
| # from "x" to {"name"=>"x"} because in "ec2:CreateLaunchConfiguration" | |
| # it is a string, but in "ec2:RunInstances" it is an object | |
| ruby { | |
| code => " | |
| profile = event.get('[requestParameters][iamInstanceProfile]') | |
| if profile.is_a? String | |
| event.set('[requestParameters][iamInstanceProfile]', { | |
| 'name' => profile | |
| }) | |
| end | |
| " | |
| } | |
| } | |
| } | |
| } | |
| output { | |
| if "metric" in [tags] { | |
| stdout { | |
| codec => line { | |
| format => "rate: %{[events][rate_1m]}" | |
| } | |
| }z | |
| } | |
| if [type] == "cloudtrail" { | |
| elasticsearch { | |
| action => update | |
| doc_as_upsert => true | |
| hosts => ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] | |
| index => "cloudtrail-%{+YYYY.MM.dd}" | |
| document_id => "%{eventID}" | |
| manage_template => false | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment