Installation from RPM

[root@NLRTM1-S0503 ~]# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
[root@NLRTM1-S0503 ~]# vi /etc/yum.repos.d/elastic.repo

[logstash-6.x]
name=Elastic repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

[elasticsearch-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

[kibana-6.x]
name=Kibana repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

[root@NLRTM1-S0503 ~]# yum install elasticsearch
[root@NLRTM1-S0503 ~]# yum install kibana
[root@NLRTM1-S0503 ~]# yum install logstash

User preparation

Logstash

WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using –path.settings. Continuing using the defaults

[root@NLRTM1-S0503 logstash]# ./bin/logstash -t
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2018-12-06 00:08:16.015 [main] writabledirectory - Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[INFO ] 2018-12-06 00:08:16.024 [main] writabledirectory - Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
ERROR: Failed to read pipelines yaml file. Location: /usr/share/logstash/config/pipelines.yml

remedy:

[support@NLRTM1-S0503 logstash]$ ./bin/logstash --path.settings /etc/logstash -t

Start logstash as a service

Check logstash service user and correct permissions

[root@NLRTM1-S0503 logstash]# vi /etc/systemd/system/logstash.service 
[root@NLRTM1-S0503 etc]# chown -R logstash:logstash /etc/logstash
[root@NLRTM1-S0503 etc]# chown -R logstash:logstash /usr/share/logstash

# chmod -R g+rwx /usr/share/logstash/
# chown -R logstash:logstash /var/log/logstash

[root@NLRTM1-S0503 logstash]# /bin/systemctl daemon-reload
[root@NLRTM1-S0503 logstash]# systemctl enable logstash.service
Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.
[root@NLRTM1-S0503 logstash]# systemctl start logstash.service

Setting up ELK components as service

[root@cilacap etc]# chown -R logstash:logstash logstash
[root@cilacap etc]# chown -R elasticsearch:elasticsearch elasticsearch
[root@cilacap etc]# chown -R kibana:kibana kibana

[root@cilacap etc]# usermod -aG logstash elastic
[root@cilacap etc]# usermod -aG elasticsearch elastic
[root@cilacap etc]# usermod -aG kibana elastic
[root@cilacap etc]# groups elastic
elastic : elastic wheel logstash elasticsearch kibana

[root@cilacap etc]# sudo /bin/systemctl daemon-reload
[root@cilacap etc]# sudo /bin/systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@cilacap etc]# systemctl enable logstash.service
Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.
[root@cilacap etc]# systemctl enable kibana.service
Created symlink from /etc/systemd/system/multi-user.target.wants/kibana.service to /etc/systemd/system/kibana.service.
[root@cilacap etc]#


# cp /etc/systemd/system/logstash-ecn4.service /etc/systemd/system/logstash-apex.service
# vi /etc/systemd/system/logstash-apex.service
# cd /etc/logstash/
# cp -R ecn4 apex
# cd apex
# vi logstash.yml 
[root@NLRTM1-S0503 logstash]# vi /etc/logstash/apex/pipelines.yml
[root@NLRTM1-S0503 logstash]# chown -R logstash:logstash apex
[root@NLRTM1-S0503 logstash]# systemctl enable logstash-apex.service 
Created symlink from /etc/systemd/system/multi-user.target.wants/logstash-apex.service to /etc/systemd/system/logstash-apex.service. 

Filebeat module APACHE2

[root@one filebeat]# ./filebeat modules enable apache2

[root@one filebeat]# nohup ./filebeat >/dev/null 2>&1 &

[root@one filebeat]# tail logs/filebeat
2018-12-08T21:06:44.246+0100	ERROR	pipeline/output.go:100	Failed to connect to backoff(elasticsearch(http://www.atikin.nl:49200)): Connection marked as failed because the onConnect callback failed: Error loading pipeline for fileset apache2/access: This module requires the following Elasticsearch plugins: ingest-user-agent, ingest-geoip. You can install them by running the following commands on all the Elasticsearch nodes:
    sudo bin/elasticsearch-plugin install ingest-user-agent
    sudo bin/elasticsearch-plugin install ingest-geoip

Install Elasticsearch plugins

[root@cilacap ~]# cd /usr/share/elasticsearch/
[root@cilacap elasticsearch]# ./bin/elasticsearch-plugin install ingest-user-agent
-> Downloading ingest-user-agent from elastic
[=================================================] 100%   
-> Installed ingest-user-agent
[root@cilacap elasticsearch]# ./bin/elasticsearch-plugin install ingest-geoip
-> Downloading ingest-geoip from elastic
[=================================================] 100%   
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@     WARNING: plugin requires additional permissions     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
* java.lang.RuntimePermission accessDeclaredMembers
* java.lang.reflect.ReflectPermission suppressAccessChecks
See http://docs.oracle.com/javase/8/docs/technotes/guides/security/permissions.html
for descriptions of what these permissions allow and the associated risks.
Continue with installation? [y/N]y
-> Installed ingest-geoip

[root@cilacap elasticsearch]# systemctl restart elasticsearch

Apache module configuration

[root@one filebeat]# vi /opt/elastic/filebeat/modules.d/apache2.yml
- module: apache2
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/httpd/paulowna_site.com-access_log*","/var/log/httpd/paulowna_shop.com-access_log*" ]

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/httpd/paulowna_site.com_error_log", "/var/log/httpd/paulowna_shop.com-error_log"]

Run filebeat

[root@one filebeat]# /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat -d "publish" 

Run filebeat in background

nohup /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat  -d "publish" 2>&1 >/dev/null &

Journalctl

[root@NLRTM1-S0503 es00]# journalctl -u elasticsearch.service