Linux 下使用 ElasticSearch + LogStash + Kibana 和 rsyslog 收集展示系统日志

文章

林里克斯

106字数

Linux,Elasticsearch,Kibana

实验平台：CentOS Linux release 7.6.1810 (Core)

ElasticSearch Version：elasticsearch-7.11.2

LogStash Version：logstash-7.11.2

Kibana Version：kibana-7.11.2

rsyslog Version：rsyslogd 8.24

一、ElasticSearch

1.安装 ElasticSearch

$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.11.2-linux-x86_64.tar.gz
$ tar xf elasticsearch-7.11.2-linux-x86_64.tar.gz
$ mv elasticsearch-7.11.2 elasticsearch
$ cd elasticsearch
#这里使用单节点测试，就使用默认配置文件启动

2.启动

#!/bin/bash

export elasticsearch_base=/data/elasticsearch
export runuser=jarbo
export JAVA_HOME=${elasticsearch_base}/jdk

export PATH=$PATH:${JAVA_HOME}/bin
export CLASSPATH=.:${JAVA_HOME}/lib/dt.jar:${JAVA_HOME}/lib/tools.jar
export JAVA_HOME JAVA_BIN PATH CLASSPATH

#if [ "${whoami}" != "${runuser}" ];then
#       echo "please use ${runuser} running"
#       exit 0
#fi

case "$1" in
start)
    ${elasticsearch_base}/bin/elasticsearch -d -p ${elasticsearch_base}/elasticsearch.pid
    echo "elasticsearch startup"
    ;;

stop)
    es_pid=`ps -ef|grep  "Des.path.home=${elasticsearch_base}" | grep -v grep | awk '{print $2}'`
    kill -9 $es_pid 2>/dev/null 1>/dev/null
    echo "elasticsearch stopped"
    ;;

status)
    es_pid=`ps -ef |grep ${runuser}|grep "Des.path.home=${elasticsearch_base}" | grep -v grep | awk '{print $2}'`
    if [[ $es_pid ]];then echo "elasticsearch is running"; else echo "elasticsearch not running"; fi
    ;;

restart)
    es_pid=`ps -ef|grep ${runuser}|grep "Des.path.home=${elasticsearch_base}" | grep -v grep | awk '{print $2}'`
    kill -9 $es_pid
    echo "elasticsearch stopped"

    ${elasticsearch_base}/bin/elasticsearch -d
    echo "elasticsearch startup"

    ;;

*)
    echo "start|stop|restart|status"
    ;;  
esac

exit $?

二、Kibana

1.安装

$ wget https://artifacts.elastic.co/downloads/kibana/kibana-7.11.2-linux-x86_64.tar.gz
$ tar xf kibana-7.11.2-linux-x86_64.tar.gz
$ mv kibana-7.11.2 kibana
$ cd kibana
$ egrep -v "^$|^#" kibana.yml 
server.port: 5601
server.host: "10.10.13.13"
elasticsearch.hosts: ["http://localhost:9200"]
i18n.locale: "zh-CN"

2.启动

#!/bin/bash
cd $(cd "$(dirname "$0")"; pwd)
SERVER_HOME=$PWD
LOG_HOME=$SERVER_HOME/logs
BIN_HOME=$SERVER_HOME/bin

START_SHELL=$BIN_HOME/kibana
PIDFILE=$BIN_HOME/server.pid
CONSOLE=$LOG_HOME/kibana-console_`date '+%Y-%m-%d'`.log
test -d $LOG_HOME || mkdir -p $LOG_HOME

start()
{
        echo "staring ..."
        nohup $START_SHELL >$CONSOLE 2>&1  &
        echo $! > $PIDFILE
}

stop()
{
if [ -f "$PIDFILE" ] ; then
    echo "kibana stoping.."    
    kill `cat $PIDFILE`
    sleep 1
    rm -rf $PIDFILE
    echo "stop kibana success"
  else
    echo "kibana is not running"
    exit 0;
  fi;
}

status()
{
    if [ -f "$PIDFILE" ] ; then
       pid=`cat $PIDFILE`
       if [[ pid -gt 0 ]]
       then
           echo "[$(date '+%Y-%m-%d %T')] kibana is running.( pid:$pid )"
       else
            echo "[$(date '+%Y-%m-%d %T')] kibana is not running"
       fi
    else 
         echo "kibana is not running"
    fi
}

restart()
{
   stop
   start
}

case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    status)
         status
        ;;
    restart)
        restart
        ;;
    *)
     echo "Usage: bash ./server.sh {start|status|stop|restart}"
        exit 1
        ;;
esac
exit 0

三、rsyslog

1.编辑配置文件

$ sudo vim /etc/rsyslog.conf
$ModLoad imtcp
$InputTCPServerRun 514

*.* @@localhost:4560

2.重启服务

$ sudo systemctl restart rsyslog
$ netstat -tlnp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      6566/sshd
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      6812/master
tcp        0      0 0.0.0.0:514             0.0.0.0:*               LISTEN      9903/rsyslogd           
tcp6       0      0 :::22                   :::*                    LISTEN      6566/sshd
tcp6       0      0 ::1:25                  :::*                    LISTEN      6812/master
tcp6       0      0 :::514                  :::*                    LISTEN      9903/rsyslogd
#监听 514 端口

四、logstash

1.安装

$ wget https://artifacts.elastic.co/downloads/logstash/logstash-7.11.2-linux-x86_64.tar.gz
$ tar xf logstash-7.11.2-linux-x86_64.tar.gz
$ mv logstash-7.11.2 logstash
$ cd logstash/config/
$ vim logstash.conf
input {
  syslog {
    type => "rsyslog"
    port => 4560
    #Rsyslog服务应用端口
  }
}

output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    #elasticsearch 的地址
    index => "rsyslog-%{+YYYY.MM}"
    #存入到 elasticsearch 的索引名称
    }
}

2.启动

$ /data/logstash/bin/logstash -f /data/logstash/config/logstash.conf &

3.使用 nginx 代理 Kibana

location / {
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto http;
    client_max_body_size 1G;
    proxy_pass http://10.10.13.13:5601;
    rewrite ^/kibana/(.*)$ /$1 break;
}