ELK install

elkinstall.sh

# https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.htmlhttps://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html

##############################################
#
# ELK Server
#
##############################################

yum install -y java
rpm --import http://packages.elasticsearch.org/GPG-KEY-elasticsearch

cat <<EOF > /etc/yum.repos.d/elasticsearch.repo
[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

yum makecache fast


yum install -y elasticsearch
systemctl start elasticsearch && systemctl enable elasticsearch

# check that elasticsearch is working
# curl -X GET "localhost:9200/_cat/health?v"


# Install Kibana
# https://www.elastic.co/guide/en/kibana/current/setup.html
yum install -y kibana

vi /etc/kibana/kibana.yml
# server.host: "10.160.16.10"

systemctl start kibana.service && systemctl enable kibana.service


yum install -y logstash
systemctl start logstash && systemctl enable logstash
# Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.
cat <<EOF > /etc/logstash/conf.d/nifi_logs.conf
input {
    beats {
        port => "5044"
    }
}
filter {
    grok {
    	break_on_match => false
    	named_captures_only => true
    	patterns_dir => ["/opt/logstash/patterns"]
        match => {
        	"message" => "%{DATESTAMP:nifiTimestamp} %{WORD:logLevel:tag} \[%{GREEDYDATA:nifiTaskInformation}\] %{NOTSPACE:nifiEventType} %{GREEDYDATA:eventText}"
        }
    }
    mutate {    	
        add_field => {
        	"received_at" => "%{@timestamp}" 
        	"received_from" => "%{host}" 
        	"raw_log" => "%{message}" 
    	}
    }
}
output {
	elasticsearch { hosts => ["localhost:9200"] }
}
EOF 


cat <<EOF > /etc/logstash/conf.d/patterns/nifi_patterns
CUSTOM_YEAR [0-9]{4}-[0-9]{2}-[0-9]{2}
NIFITIMESTAMP %{CUSTOM_YEAR} %{TIME}
NIFI_TASK \[(.*)\]
NIFI_NOSPACE_BEGINNING ^(\S*)
EOF 

cat <<EOF > /etc/logstash/logstash.yml
path.data: /var/lib/logstash
path.logs: /var/log/logstash

http.host: "10.160.16.28"
EOF 

chown -R logstash /etc/logstash/

# sudo /usr/share/logstash/bin/logstash --config.reload.automatic "--path.settings" "/etc/logstash/"    ## important !! the last slash is not present
# Check the service while running: sudo journalctl -f -u logstash

##############################################
#
# Filebeat clients
#
##############################################
rpm --import http://packages.elasticsearch.org/GPG-KEY-elasticsearch

cat <<EOF > /etc/yum.repos.d/elasticsearch.repo
[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
EOF

#
# Filebeat
#
yum makecache fast
yum install -y filebeat

rm /etc/filebeat/filebeat.yml
cat <<EOF > /etc/filebeat/filebeat.yml
filebeat.prospectors:
- type: log
  paths:
    - /opt/nifi/logs/nifi-app.log
output.logstash:
  hosts: ["10.160.16.28:5044"]
EOF

systemctl start filebeat && systemctl enable filebeat

#
# Metricbeat
#
yum install -y metricbeat

rm /etc/metricbeat/metricbeat.yml
cat <<EOF > /etc/metricbeat/metricbeat.yml
tags: ["service-X", "web-tier"]
env: staging

filebeat.prospectors:
- type: log
  paths:
    - /opt/nifi/logs/nifi-app.log
output.logstash:
  hosts: ["10.160.16.28:5044"]
  compression_level: 1
EOF