Linux环境EFK+Nginx+Kafka日志平台自动化安装脚本

小柒助手 Shell评论79,5811字数 3122阅读10分24秒阅读模式

此脚本是Linux一键部署ELK日志平台自动化脚本,有需要朋友可以参考,脚本内容如下:

环境准备

操作系统:CentOS Linux release 7.8.2003文章源自小柒网-https://www.yangxingzhen.cn/7533.html

软件版本

elasticsearch:elasticsearch-7.5.1-linux-x86_64.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

kibana:kibana-7.5.1-linux-x86_64.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

logstash:logstash-7.5.1.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

filebeat:filebeat-7.5.1-linux-x86_64.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

JDK:jdk-11.0.1_linux-x64_bin.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

Nginx:nginx-1.18.0.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

Zookeeper:zookeeper-3.4.10.tar.gz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

Kafka:kafka_2.12-2.5.0.tgz文章源自小柒网-https://www.yangxingzhen.cn/7533.html

脚本功能

1)一键安装Elasticsearch、Kibana、Logstash、Filebeat文章源自小柒网-https://www.yangxingzhen.cn/7533.html

2)一键安装Zookeeper

3)一键安装Kafka

4)一键安装Nginx

5)自动添加nginx_access、nginx_error索引

6)自动配置Elasticsearch用户密码

[root@localhost ~]# vim install_elk_filebeat_kafka.sh

#!/bin/bash
#Date:2019-5-20 13:14:00
#Author Blog:
#	https://www.yangxingzhen.cn
#   https://www.i7ti.cn
#Author WeChat:
#	微信公众号:小柒博客
#Author mirrors site:
#	https://mirrors.yangxingzhen.com
#About the Author
#	BY:YangXingZhen
#	Mail:xingzhen.yang@yangxingzhen.com
#   QQ:675583110

User="elk"
Elasticsearch_User="elastic"
Elasticsearch_Passwd="www.yangxingzhen.cn"
IPADDR=$(hostname -I |awk '{print $1}')
Elasticsearch_DIR="/data/elasticsearch"
Kafka_IP=$(hostname -I |awk '{print $1}')
Zookeeper_IP=$(hostname -I |awk '{print $1}')
Elasticsearch_IP=$(hostname -I |awk '{print $1}')

# Define JDK path variables
JDK_URL=https://mirrors.yangxingzhen.com/jdk
JDK_File=jdk-11.0.1_linux-x64_bin.tar.gz
JDK_File_Dir=jdk-11.0.1
JDK_Dir=/usr/local/jdk-11.0.1

# Define Zookeeper path variables
Zookeeper_URL=http://archive.apache.org/dist/zookeeper/zookeeper-3.4.10
Zookeeper_File=zookeeper-3.4.10.tar.gz
Zookeeper_File_Dir=zookeeper-3.4.10
Zookeeper_PREFIX=/usr/local/zookeeper

# Define Kafka path variables
Kafka_URL=https://archive.apache.org/dist/kafka/2.5.0
Kafka_File=kafka_2.12-2.5.0.tgz
Kafka_File_Dir=kafka_2.12-2.5.0
Kafka_Dir=/usr/local/kafka

# Define Nginx path variables
Nginx_URL=http://nginx.org/download
Nginx_File=nginx-1.18.0.tar.gz
Nginx_File_Dir=nginx-1.18.0
Nginx_Dir=/usr/local/nginx

# Define Elasticsearch path variables
Elasticsearch_URL=https://artifacts.elastic.co/downloads/elasticsearch
Elasticsearch_File=elasticsearch-7.5.1-linux-x86_64.tar.gz
Elasticsearch_File_Dir=elasticsearch-7.5.1
Elasticsearch_Dir=/usr/local/elasticsearch

# Define Logstash path variables
Filebeat_URL=https://artifacts.elastic.co/downloads/beats/filebeat
Filebeat_File=filebeat-7.5.1-linux-x86_64.tar.gz
Filebeat_File_Dir=filebeat-7.5.1-linux-x86_64
Filebeat_Dir=/usr/local/filebeat

# Define Kafka path variables
Logstash_URL=https://artifacts.elastic.co/downloads/logstash
Logstash_File=logstash-7.5.1.tar.gz
Logstash_File_Dir=logstash-7.5.1
Logstash_Dir=/usr/local/logstash

# Define Kibana path variables
Kibana_URL=https://artifacts.elastic.co/downloads/kibana
Kibana_File=kibana-7.5.1-linux-x86_64.tar.gz
Kibana_File_Dir=kibana-7.5.1-linux-x86_64
Kibana_Dir=/usr/local/kibana

# 配置内核参数
cat >>/etc/security/limits.conf <<EOF
* soft nofile 65537
* hard nofile 65537
* soft nproc 65537
* hard nproc 65537
EOF

if [ $(grep -wc "4096" /etc/security/limits.d/20-nproc.conf) -eq 0 ];then
cat >>/etc/security/limits.d/20-nproc.conf <<EOF
*          soft    nproc     4096
EOF
fi

cat >/etc/sysctl.conf <<EOF
net.ipv4.tcp_max_syn_backlog = 65536
net.core.netdev_max_backlog = 32768
net.core.somaxconn = 32768
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_fin_timeout = 120
net.ipv4.tcp_keepalive_time = 120
net.ipv4.ip_local_port_range = 1024 65535 
net.ipv4.tcp_max_tw_buckets = 30000
fs.file-max=655350
vm.max_map_count = 262144
net.core.somaxconn= 65535
net.ipv4.ip_forward = 1
net.ipv6.conf.all.disable_ipv6=1
EOF

# sysctl -p使其配置生效
sysctl -p >/dev/null

# 创建elk用户
[ $(grep -wc "elk" /etc/passwd) -eq 0 ] && useradd elk >/dev/null

# 安装JDK环境
java -version >/dev/null 2>&1
if [ $? -ne 0 ];then
	# Install Package
	[ -f /usr/bin/wget ] || yum -y install wget >/dev/null
	wget -c ${JDK_URL}/${JDK_File}
	tar xf ${JDK_File}
	mv ${JDK_File_Dir} ${JDK_Dir}
cat >>/etc/profile <<EOF
export JAVA_HOME=${JDK_Dir}
export CLASSPATH=\$CLASSPATH:\$JAVA_HOME/lib:\$JAVA_HOME/jre/lib
export PATH=\$JAVA_HOME/bin:\$JAVA_HOME/jre/bin:\$PATH:\$HOMR/bin
EOF
fi

# 加载环境变量
source /etc/profile >/dev/null

# Install Zookeeper
if [ -d ${Zookeeper_PREFIX} ];then
	echo -e "\033[31mThe Zookeeper Already Install...\033[0m"
	exit 1
else
	cd ~ && wget -c ${Zookeeper_URL}/${Zookeeper_File}
	tar xf ${Zookeeper_File}
	\mv ${Zookeeper_File_Dir} ${Zookeeper_PREFIX}
	chown -R root.root ${Zookeeper_PREFIX}
	mkdir -p ${Zookeeper_PREFIX}/{data,logs}
	\cp ${Zookeeper_PREFIX}/conf/zoo_sample.cfg ${Zookeeper_PREFIX}/conf/zoo.cfg
cat >${Zookeeper_PREFIX}/conf/zoo.cfg <<EOF
#服务器之间或客户端与服务器之间的单次心跳检测时间间隔,单位为毫秒
tickTime=2000
#集群中的follower服务器(F)与leader服务器(L)之间初始连接时能容忍的最多心跳数(tickTime的数量)
initLimit=10
#集群中flower服务器(F)跟leader(L)服务器之间的请求和答应最多能容忍的心跳数
syncLimit=5
#客户端连接Zookeeper服务器的端口,Zookeeper会监听这个端口,接受客户端的访问请求
clientPort=2181
#存放数据文件
dataDir=${Zookeeper_PREFIX}/data
#存放日志文件
dataLogDir=${Zookeeper_PREFIX}/logs
#Zookeeper cluster,2888为选举端口,3888为心跳端口
#服务器编号=服务器IP:LF数据同步端口:LF选举端口
server.1=${IPADDR}:2888:3888
EOF

# 写入服务ID编号
echo "1" > ${Zookeeper_PREFIX}/data/myid

# Add power on self start And Start Zookeeper
source /etc/profile >/dev/null && ${Zookeeper_PREFIX}/bin/zkServer.sh start
fi

# Install Kafka Soft
if [ ! -d ${Kafka_Dir} ];then
	cd ~ && wget -c ${Kafka_URL}/${Kafka_File}
	tar xf ${Kafka_File}
	mv ${Kafka_File_Dir} ${Kafka_Dir}
# 编辑配置文件
cat >${Kafka_Dir}/config/server.properties <<EOF
listeners=PLAINTEXT://${IPADDR}:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=10
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=${IPADDR}:2181
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=0	
EOF

# 判断Zookeeper服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待Zookeeper服务启动...\033[0m"
	# 获取Zookeeper服务端口
	[ -f /usr/bin/netstat ] || yum -y install net-tools >/dev/null
	netstat -lntup |grep "2181" >/dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done

# 启动Kafka服务
source /etc/profile >/dev/null && ${Kafka_Dir}/bin/kafka-server-start.sh -daemon ${Kafka_Dir}/config/server.properties

# 判断Kafka服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待Kafka服务启动...\033[0m"
	# 获取Kafka服务端口
	netstat -lntup |grep "9092" >/dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done

else
	echo -e "\033[31mThe Kafka Already Install...\033[0m"
	exit 1
fi

# Install Elasticsearch
if [ ! -d ${Elasticsearch_Dir} ];then
	# Install Package
	[ -f /usr/bin/wget ] || yum -y install wget >/dev/null
	cd ~ && wget -c ${Elasticsearch_URL}/${Elasticsearch_File}
	tar xf ${Elasticsearch_File}
	mv ${Elasticsearch_File_Dir} ${Elasticsearch_Dir}
else
	echo -e "\033[31mThe Elasticsearch Already Install...\033[0m"
	exit 1
fi

# Install Kibana
if [ ! -d ${Kibana_Dir} ];then
	# Install Package
	[ -f /usr/bin/wget ] || yum -y install wget >/dev/null
	cd ~ && wget -c ${Kibana_URL}/${Kibana_File}
	tar xf ${Kibana_File}
	mv ${Kibana_File_Dir} ${Kibana_Dir}
else
	echo -e "\033[31mThe Kibana Already Install...\033[0m"
	exit 1
fi 

# 配置Elasticsearch
mkdir -p ${Elasticsearch_DIR}/{data,logs}
cat >${Elasticsearch_Dir}/config/elasticsearch.yml <<EOF
# 节点名称
node.name: es-master
# 存放数据目录,先创建该目录
path.data: ${Elasticsearch_DIR}/data
# 存放日志目录,先创建该目录
path.logs: ${Elasticsearch_DIR}/logs
# 节点IP
network.host: ${Elasticsearch_IP}
# tcp端口
transport.tcp.port: 9300
# http端口
http.port: 9200
# 主合格节点列表,若有多个主节点,则主节点进行对应的配置
cluster.initial_master_nodes: ["${Elasticsearch_IP}:9300"]
# 是否允许作为主节点
node.master: true
# 是否保存数据
node.data: true
node.ingest: false
node.ml: false
cluster.remote.connect: false
# 跨域
http.cors.enabled: true
http.cors.allow-origin: "*"
# 配置X-Pack
http.cors.allow-headers: Authorization
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
EOF

# 配置Kibana
cat >${Kibana_Dir}/config/kibana.yml <<EOF
server.port: 5601
server.host: "${Elasticsearch_IP}"
elasticsearch.hosts: ["http://${Elasticsearch_IP}:9200"]
elasticsearch.username: "${Elasticsearch_User}"
elasticsearch.password: "${Elasticsearch_Passwd}"
logging.dest: ${Kibana_Dir}/logs/kibana.log
i18n.locale: "zh-CN"
EOF

# 创建Kibana日志目录
[ -d ${Kibana_Dir}/logs ] || mkdir ${Kibana_Dir}/logs

# 授权ELK用户管理Elasticsearch、Kibana
chown -R ${User}.${User} ${Elasticsearch_Dir}
chown -R ${User}.${User} ${Elasticsearch_DIR}
chown -R root.root ${Kibana_Dir}

# 创建systemctl管理配置文件
cat >/usr/lib/systemd/system/elasticsearch.service <<EOF
[Unit]
Description=elasticsearch
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
LimitCORE=infinity
LimitNOFILE=655360
LimitNPROC=655360
User=${User}
Group=${User}
PIDFile=${Elasticsearch_Dir}/logs/elasticsearch.pid
ExecStart=${Elasticsearch_Dir}/bin/elasticsearch
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID
RestartSec=30
Restart=always
PrivateTmp=true

[Install]
WantedBy=multi-user.target
EOF

# 启动Elasticsearch服务
systemctl daemon-reload
systemctl enable elasticsearch
systemctl start elasticsearch

# 判断Elasticsearch服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待Elasticsearch服务启动...\033[0m"
	# 获取Elasticsearch服务端口
	netstat -lntup |egrep "9200|9300" >/dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done

# 生成Elasticsearch密码
cat >/tmp/config_elasticsearch_passwd.exp <<EOF
spawn su ${User} -c "source /etc/profile >/dev/null && ${Elasticsearch_Dir}/bin/elasticsearch-setup-passwords interactive"
set timeout 60
expect {
        -timeout 20
        "y/N" {
                send "y\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Enter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
        "Reenter password *:" {
                send "${Elasticsearch_Passwd}\n"
                exp_continue
                }
}
EOF

[ -f /usr/bin/expect ] || yum -y install expect >/dev/null
expect /tmp/config_elasticsearch_passwd.exp

# 创建systemctl管理配置文件
cat >/usr/lib/systemd/system/kibana.service <<EOF
[Unit]
Description=kibana
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
PIDFile=/var/run/kibana.pid
ExecStart=/usr/local/kibana/bin/kibana --allow-root
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID
PrivateTmp=false

[Install]
WantedBy=multi-user.target
EOF

# 启动Kibana
systemctl daemon-reload
systemctl enable kibana
systemctl start kibana

# 判断Kibana服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待Kibana服务启动...\033[0m"
	# 获取Kibana服务端口
	CODE=$(curl -s -w "%{http_code}" -o /dev/null http://${IPADDR}:5601/login)
	if [ ${CODE} -eq 200 ];then
		Code="break"
	fi
${Code}
done

# Install Filebeat
if [ ! -d ${Filebeat_Dir} ];then
	cd ~ && wget -c ${Filebeat_URL}/${Filebeat_File}
	tar xf ${Filebeat_File}
	mv ${Filebeat_File_Dir} ${Filebeat_Dir}
else
	echo -e "\033[31mThe Filebeat Already Install...\033[0m"
	exit 1
fi

# Install Logstash
if [ ! -d ${Logstash_Dir} ];then
	cd ~ && wget -c ${Logstash_URL}/${Logstash_File}
	tar xf ${Logstash_File}
	mv ${Logstash_File_Dir} ${Logstash_Dir}
else
	echo -e "\033[31mThe Logstash Already Install...\033[0m"
	exit 1
fi 

# Install Nginx Soft
if [ ! -d ${Nginx_Dir} ];then
	# Install Package
	yum -y install pcre pcre-devel openssl openssl-devel gcc gcc-c++
	cd ~ && wget -c ${Nginx_URL}/${Nginx_File}
	tar zxf ${Nginx_File}
	cd ${Nginx_File_Dir}
	sed -i 's/1.18.0/ /;s/nginx\//nginx/' src/core/nginx.h
	useradd -s /sbin/nologin www
	./configure --prefix=${Nginx_Dir} \
	--user=www \
	--group=www \
	--with-http_ssl_module \
	--with-http_stub_status_module \
	--with-stream
	if [ $? -eq 0 ];then
		make -j$(nproc) && make install
		echo -e "\033[32mThe Nginx Install Success...\033[0m"
	else
		echo -e "\033[31mThe Nginx Install Failed...\033[0m"
		exit 1
	fi
else
	echo -e "\033[31mThe Nginx already Install...\033[0m"
	exit 1
fi

#Config Nginx
ln -sf ${Nginx_Dir}/sbin/nginx /usr/sbin
cat >${Nginx_Dir}/conf/nginx.conf <<EOF
	user www www;
	worker_processes auto;
	pid /usr/local/nginx/logs/nginx.pid;
events {
	use epoll;
	worker_connections 10240;
	multi_accept on;
	}
http	{
	include       mime.types;
	default_type  application/octet-stream;
	log_format json '{"@timestamp":"\$time_iso8601",'
			'"host":"\$server_addr",'
			'"clientip":"\$remote_addr",'
			'"remote_user":"\$remote_user",'
			'"request":"\$request",'
			'"http_user_agent":"\$http_user_agent",'
			'"size":\$body_bytes_sent,'
			'"responsetime":\$request_time,'
			'"upstreamtime":"\$upstream_response_time",'
			'"upstreamhost":"\$upstream_addr",'
			'"http_host":"\$host",'
			'"requesturi":"\$request_uri",'
			'"url":"\$uri",'
			'"domain":"\$host",'
			'"xff":"\$http_x_forwarded_for",'
			'"referer":"\$http_referer",'
			'"status":"\$status"}';
	access_log logs/access.log json;
	error_log logs/error.log warn;
	sendfile        on;
	tcp_nopush          on;
	keepalive_timeout  120;
	tcp_nodelay         on;
	server_tokens off;
	gzip    on;
	gzip_min_length 1k;
	gzip_buffers    4 64k;
	gzip_http_version 1.1;
	gzip_comp_level 4;
	gzip_types      text/plain application/x-javascript text/css application/xml;
	gzip_vary       on;
	client_max_body_size 10m;
	client_body_buffer_size 128k;
	proxy_connect_timeout 90;
	proxy_send_timeout 90;
	proxy_buffer_size 4k;
	proxy_buffers 4 32k;
	proxy_busy_buffers_size 64k;
	large_client_header_buffers 4 4k;
	client_header_buffer_size 4k;
	open_file_cache_valid 30s;
	open_file_cache_min_uses 1;
server {
	listen	80;
	server_name	localhost;
	location / {
	proxy_pass   http://${IPADDR}:5601;
	proxy_set_header   Host             \$host;
	proxy_set_header   X-Real-IP        \$remote_addr;
	proxy_set_header   X-Forwarded-For  \$proxy_add_x_forwarded_for;
	}
  }
}
EOF

# 创建systemctl管理配置文件
cat >/usr/lib/systemd/system/nginx.service <<EOF
[Unit]
Description=Nginx Server
Documentation=http://nginx.org/en/docs/
After=network-online.target remote-fs.target nss-lookup.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=${Nginx_Dir}/logs/nginx.pid
ExecStart=${Nginx_Dir}/sbin/nginx -c ${Nginx_Dir}/conf/nginx.conf
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID

[Install]
WantedBy=multi-user.target
EOF

# Start Nginx
systemctl daemon-reload
systemctl enable nginx
systemctl start nginx

# 配置Filebeat
cat >${Filebeat_Dir}/filebeat.yml <<EOF
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - ${Nginx_Dir}/logs/access.log
  multiline:
      pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
      negate: true
      match: after
  fields:
    log_topics: nginx_access-log
    logtype: nginx_access
- type: log
  enabled: true
  paths:
    - ${Nginx_Dir}/logs/error.log
  multiline:
      pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
      negate: true
      match: after
  fields:
    log_topics: nginx_error-log
    logtype: nginx_error
output.kafka:
  enabled: true
  hosts: ["${Kafka_IP}:9092"]
  topic: '%{[fields][log_topics]}'
EOF

# 配置Logstash
cat >${Logstash_Dir}/config/nginx.conf <<EOF
input {
    kafka {
        bootstrap_servers => "${Kafka_IP}:9092"
        group_id => "logstash-group"
        topics => ["nginx_access-log","nginx_error-log"]
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        codec => json
    }
}

filter {
    if [fields][logtype] == "nginx_access" {
        json {
            source => "message"
        } 
		
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}" }
        }
		
        date {
            match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
            target => "@timestamp"
        }
    }
	if [fields][logtype] == "nginx_error" {
        json {
            source => "message"
        } 
		
        grok {
            match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level}" }
        }
		
        date {
            match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
            target => "@timestamp"
        }
    }
}

output {
    if [fields][logtype] == "nginx_access" {
        elasticsearch {
            hosts => ["${Elasticsearch_IP}:9200"]
            user => "${Elasticsearch_User}"
            password => "${Elasticsearch_Passwd}"
            action => "index"
            index => "nginx_access.log-%{+YYYY.MM.dd}"
        }
    }
    if [fields][logtype] == "nginx_error" {
        elasticsearch {
            hosts => ["${Elasticsearch_IP}:9200"]
            user => "${Elasticsearch_User}"
            password => "${Elasticsearch_Passwd}"
            action => "index"
            index => "nginx_error.log-%{+YYYY.MM.dd}"
        }
    }	
}
EOF

# 创建Filebeat日志目录
[ -d ${Filebeat_Dir}/logs ] || mkdir ${Filebeat_Dir}/logs

# 授权ELK用户管理Filebeat、Logstash
chown -R ${User}.${User} ${Filebeat_Dir}
chown -R ${User}.${User} ${Logstash_Dir}

# 启动Filebeat
su ${User} -c "cd ${Filebeat_Dir} && nohup ./filebeat -e -c filebeat.yml >>${Filebeat_Dir}/logs/filebeat.log >/dev/null 2>&1 &"

# 启动Logstash
su ${User} -c "cd ${Logstash_Dir}/bin && nohup ./logstash -f ${Logstash_Dir}/config/nginx.conf >/dev/null 2>&1 &"

# 判断Logstash服务是否启动,启动成功才执行以下操作
Code=""
while sleep 10
do
	echo -e "\033[32m$(date +'%F %T') 等待Logstash服务启动...\033[0m"
	# 获取Logstash服务端口
	netstat -lntup |grep "9600" >/dev/null
	if [ $? -eq 0 ];then
		Code="break"
	fi
${Code}
done

echo -e "\033[32mELK日志分析平台搭建完毕... \n通过浏览器访问:http://${IPADDR}\n用户名:elastic\n密码:www.yangxingzhen.cn\033[0m"

脚本执行方式:

[root@localhost ~]# sh install_elk_filebeat_kafka.sh

脚本执行过程截图如下

若文章图片、下载链接等信息出错,请在评论区留言反馈,博主将第一时间更新!如本文“对您有用”,欢迎随意打赏,谢谢!

继续阅读
Wechat
微信扫一扫,加我!
weinxin
我的微信
微信号已复制
微信公众号
微信扫一扫,关注我!
weinxin
我的公众号
公众号已复制
Shell最后更新:2022-12-6
小柒助手
  • 本文由 小柒助手 发表于 2020年11月13日 10:04:43
  • 声明:本站所有文章,如无特殊说明或标注,本站文章均为原创。任何个人或组织,在未征得本站同意时,禁止复制、盗用、采集、发布本站内容到任何网站、书籍等各类媒体平台。转载请务必保留本文链接:https://www.yangxingzhen.cn/7533.html
匿名

发表评论

匿名网友
:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

拖动滑块以完成验证