ELK Installation

Prerequisite

1. sudo apt update
2. sudo apt install openjdk-8-jdk
3. sudo apt-get install -y nginx
4. sudo systemctl enable nginx
5. You Should be having sudo permission 
6. Run below command without root user (don't use sudo su to run below commands). You can run a command whoami to check that you are not logged in as root user.

Install Elastic Search

1. wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.2.0-amd64.deb
2. sudo dpkg -i elasticsearch-7.2.0-amd64.deb

Install kibana

1. sudo wget https://artifacts.elastic.co/downloads/kibana/kibana-7.2.0-amd64.deb
2. sudo dpkg -i kibana-7.2.0-amd64.deb

Install Logstash

1. sudo wget https://artifacts.elastic.co/downloads/logstash/logstash-7.2.0.deb
2. sudo dpkg -i logstash-7.2.0.deb

Install Dependencies

1. sudo apt-get install -y apt-transport-https

Install FileBeat

1. wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.2.0-amd64.deb
2. sudo dpkg -i filebeat-7.2.0-amd64.deb

Modify elasticsearch yaml file

1. sudo vi /etc/elasticsearch/elasticsearch.yml
2. Make below changes in this file

cluster.name: my-application

node.name: node-1

http.port: 9200

network.host: localhost

3. sudo systemctl start elasticsearch

Modify Kibana yaml file

1. sudo vi /etc/kibana/kibana.yml
2. Make below changes in the file

server.port: 5601

  server.host: "localhost"

3. sudo systemctl start kibana

4. sudo apt-get install -y apache2-utils

5. sudo htpasswd -c /etc/nginx/htpasswd.users kibadmin

6. sudo vi /etc/nginx/sites-available/default

  server {
    listen 80;

    server_name 3.108.42.168;

    auth_basic "Restricted Access";
    auth_basic_user_file /etc/nginx/htpasswd.users;

    location / {
        proxy_pass http://localhost:5601;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
    }
}

7. sudo systemctl restart nginx

Download Some sample data

1.sudo wget https://logz.io/sample-data

 2. sudo mv sample-data apache.log

 3. vi /etc/logstash/conf.d/apache.conf

input {
 file {
   path => "/home/ubuntu/apache.log"
   start_position => "beginning"
   sincedb_path => "/dev/null"
 }
}
filter {
      grok {
         match => {  "message" => "%{COMBINEDAPACHELOG}" }
       }
      date {
         match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
      }
      geoip {
         source => "clientip"
       }
    }
output {
  elasticsearch {
  hosts => ["localhost:9200"]
  index => "petclinic-prd-1"
}
}

4. sudo systemctl restart logstash

Getting data from filebeat

1. sudo filebeat modules list
2. sudo filebeat modules enable nginx
3. sudo filebeat modules enable system
4. cd /etc/filebeat/modules.d/
5. sudo vi nginx.yml

 access:
    enabled: true
    var.paths: ["/var/log/nginx/access.log*"]

  # Error logs
  error:
    enabled: true
    var.paths: ["/var/log/nginx/error.log*"]

6. sudo vi system.yml

- module: system
  # Syslog
  syslog:
    enabled: true
    var.paths: ["/var/log/syslog*"]
   
  auth:
    enabled: true
    var.paths: ["/var/log/auth.log*"]

7. sudo systemctl restart filebeat
8. sudo systemctl restart logstash