logstash conf template

logstash.conf.erb

input {
  tcp {
      port => 5000
      type => "raw-tcp"
    }
  courier {
    transport        => "tls"
    port             => <%= @log_courier_port %>
    ssl_certificate  => "<%= @log_courier_certpath %>"
    ssl_key          => "<%= @log_courier_keypath %>"
  }
  udp {
    port => 5000
    type => "raw-udp"
  }
}

filter {
  mutate {
    add_field => { "log_ingester" => "<%= node['hostname'] %>"}
  }
  
  if [type] == "syslog" {
    grok {
      match => {"message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:log_message}"}
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{host}" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@source_host", "%{syslog_hostname}" ]
        replace => [ "@message", "%{log_message}" ]
      }
    }
    mutate {
      remove_field => [ "syslog_hostname", "log_message", "syslog_timestamp" ]
    }
  }  # end syslog
  
 <SNIP remove sensitive filters> 

  else if [type] == "ossec" {
    grok {
      match => {"message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_host} %{DATA:syslog_program}: Alert Level: %{BASE10NUM:Alert_Level}; Rule: %{BASE10NUM:Rule} - %{GREEDYDATA:Description}; Location: \(%{HOSTNAME:reporting_host}\) %{IP:reporting_ip}->%{PATH:reporting_path}; %{GREEDYDATA:details}"}
      add_field => [ "received_at", "%{@timestamp}" ]
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
    }
    if [details] {
      grok {
        match => {"details" => "srcip: %{IP:remote_addr}; %{GREEDYDATA}"}
      }
    }
  }  # end ossec

  <SNIP remove other sensitive filters>

  else if [type] == "apache-error" {
    grok {
      match => {"message" => "\[%{DAY:day} %{SYSLOGTIMESTAMP:syslog_timestamp} %{YEAR:year}\] \[%{LOGLEVEL:level}\] ?(\[client %{IP:remote_addr}\])? %{GREEDYDATA:log_message}"}
      add_field => [ "datetime", "%{year} %{syslog_timestamp}"]
      add_field => [ "received_at", "%{@timestamp}" ]
    }
    date {
      match => ["datetime", "YYYY MMM d HH:mm:ss", "YYYY MMM dd HH:mm:ss"]
    }
  }  # end apache_error
  
}

output {
  
  statsd {
    host => "%{statsd_host}"
    sender => "%{log_ingester}"
    increment => [
      "host.%{host}.%{type}"
    ]
   count => {
   
   }
  }

  elasticsearch {
    hosts => [elasticsearch_hosts]
    workers => 2
    index => "%{type}-%{+YYYY.MM.dd}"
    codec => "plain"
    manage_template => true
    template_name => "logstash"
    template_overwrite => false
    flush_size => 100
    idle_flush_time => 1
  }

  # TODO: REPLACE HIPCHAT HERE WITH SLACK OUTPUT PLUGIN
  if [type] == "nginx-error" and !("_grokparsefailure" in [tags]) and "110: Connection timed out" in [log_message] {
    hipchat {
      color => "red"
      from => "nginx-error"
      room_id => "DEFUNCT"
      token => "DEFUNCT"
      format => "%{[datetime]}: %{[subdomain]} on %{[host]} %{[request]}: %{[log_message]}"
    }
  }
}