关于shell:Shell集群部署ELKZookeeperKafkafilebeat

 服务器 2 核 4G

filebat->kafka->logstash->elasticsearch->kibana

Elasticsearch

Node-1

#!/bin/bash
# 用户 / 明码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf <<EOF
net.ipv4.ip_forward=1
EOF
cat > /etc/sysctl.conf <<EOF
vm.max_map_count=262144
EOF
cat > /etc/security/limits.conf <<EOF
* hard nofile 65536
* soft nofile 65536
* hard nproc 65536
* soft nproc 65536
EOF
systemctl restart network;sudo sysctl -p;
ulimit -S -n;ulimit -H -n;
# ELK 用户
if id -u ${ELK_USER} >/dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [! -f /tmp/$file]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创立目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到 /usr/local/es/config/certs/
将 http.p12 和 elastic-certificates.p12 上传到 /usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-1
# ES 默认 只容许本地 127.0.0.1 和 [::1] 拜访
# 也能够设置成 0.0.0.0 容许所有 IP 主机拜访
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入正告然而能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会降级为异样,ES 无奈正确启动。network.host: 0.0.0.0
# 默认 ES 节点端口 9200，如果是伪集群（在一台服务器上搭建集群），须要批改。http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否应用内存替换分区
bootstrap.memory_lock: false
# 设置数据寄存门路
path.data: /usr/local/es/data/
# 设置日志寄存门路
path.logs: /usr/local/es/logs/
# 发现其余节点主机配置 这里配置的是 ES 所在服务器的公网 IP 地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点能够被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#减少新的参数，为了让 elasticsearch-head 插件能够拜访 es (5.x 版本，如果没有能够本人手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置 JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
    #  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此过程能够关上的最大文件数
LimitNOFILE=65535
# 指定此过程能够关上的最大过程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0- 永不超时
TimeoutStopSec=0
# SIGTERM 是进行 java 过程的信号
KillSignal=SIGTERM
# 信号只发送给给 JVM
KillMode=process
# java 过程不会被杀掉
SendSIGKILL=no
# 失常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es

Node-2

#!/bin/bash
# 用户 / 明码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf <<EOF
net.ipv4.ip_forward=1
EOF
cat > /etc/sysctl.conf <<EOF
vm.max_map_count=262144
EOF
cat > /etc/security/limits.conf <<EOF
* hard nofile 65536
* soft nofile 65536
* hard nproc 65536
* soft nproc 65536
EOF
systemctl restart network;sudo sysctl -p;
ulimit -S -n;ulimit -H -n;
# ELK 用户
if id -u ${ELK_USER} >/dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [! -f /tmp/$file]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创立目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到 /usr/local/es/config/certs/
将 http.p12 和 elastic-certificates.p12 上传到 /usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-2
# ES 默认 只容许本地 127.0.0.1 和 [::1] 拜访
# 也能够设置成 0.0.0.0 容许所有 IP 主机拜访
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入正告然而能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会降级为异样,ES 无奈正确启动。network.host: 0.0.0.0
# 默认 ES 节点端口 9200，如果是伪集群（在一台服务器上搭建集群），须要批改。http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否应用内存替换分区
bootstrap.memory_lock: false
# 设置数据寄存门路
path.data: /usr/local/es/data/
# 设置日志寄存门路
path.logs: /usr/local/es/logs/
# 发现其余节点主机配置 这里配置的是 ES 所在服务器的公网 IP 地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点能够被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#减少新的参数，为了让 elasticsearch-head 插件能够拜访 es (5.x 版本，如果没有能够本人手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置 JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
#  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此过程能够关上的最大文件数
LimitNOFILE=65535
# 指定此过程能够关上的最大过程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0- 永不超时
TimeoutStopSec=0
# SIGTERM 是进行 java 过程的信号
KillSignal=SIGTERM
# 信号只发送给给 JVM
KillMode=process
# java 过程不会被杀掉
SendSIGKILL=no
# 失常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es

Node-3

#!/bin/bash
# 用户 / 明码
ELK_USER=es
ELK_USER_PASSWORD=123
install_es(){
    yum install -y wget sudo vim
cat >> /usr/lib/sysctl.d/00-system.conf <<EOF
net.ipv4.ip_forward=1
EOF
cat > /etc/sysctl.conf <<EOF
vm.max_map_count=262144
EOF
cat > /etc/security/limits.conf <<EOF
* hard nofile 65536
* soft nofile 65536
* hard nproc 65536
* soft nproc 65536
EOF
systemctl restart network;sudo sysctl -p;
ulimit -S -n;ulimit -H -n;
# ELK 用户
if id -u ${ELK_USER} >/dev/null 2>&1; then
    echo "user exists"
else
    echo "user does not exist"
    useradd ${ELK_USER}
    echo "${ELK_USER_PASSWORD}" | passwd --stdin ${ELK_USER}
fi
file="elasticsearch-7.17.0-linux-x86_64.tar.gz"
if [! -f /tmp/$file]; then
    wget -P /tmp https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.0-linux-x86_64.tar.gz
fi
tar -zxvf /tmp/elasticsearch-7.17.0-linux-x86_64.tar.gz -C /usr/local
mv /usr/local/elasticsearch-7.17.0 /usr/local/es
# 创立目录上传证书
mkdir -p /usr/local/es/config/certs/
# 上传证书到 /usr/local/es/config/certs/
将 http.p12 和 elastic-certificates.p12 上传到 /usr/local/es/config/certs/
#链接：https://pan.baidu.com/s/1jlh5MYfFouemzF0XAqQf7w 提取码：3no1 
cat > /usr/local/es/config/elasticsearch.yml << "EOF"
# 集群名称
cluster.name: es-cluster
# 集群节点名称
node.name: node-3
# ES 默认 只容许本地 127.0.0.1 和 [::1] 拜访
# 也能够设置成 0.0.0.0 容许所有 IP 主机拜访
# 如果不配置就默认认为是开发者模式，如果配置不正确会写入正告然而能正确运行
# 如果配置了就会认为进入了生产环境, 如果配置不正确就会降级为异样,ES 无奈正确启动。network.host: 0.0.0.0
# 默认 ES 节点端口 9200，如果是伪集群（在一台服务器上搭建集群），须要批改。http.port: 9200
# 对外通信端口
transport.port: 9300
# 是否应用内存替换分区
bootstrap.memory_lock: false
# 设置数据寄存门路
path.data: /usr/local/es/data/
# 设置日志寄存门路
path.logs: /usr/local/es/logs/
# 发现其余节点主机配置 这里配置的是 ES 所在服务器的公网 IP 地址
discovery.seed_hosts: ["10.0.0.21:9300","10.0.0.22:9300","10.0.0.23:9300"]
# 哪些节点能够被选举为主节点配置
cluster.initial_master_nodes: ["10.0.0.21","10.0.0.22","10.0.0.23"]
#减少新的参数，为了让 elasticsearch-head 插件能够拜访 es (5.x 版本，如果没有能够本人手动加)
http.cors.enabled: true
http.cors.allow-origin: "*"
# 数据采集指标
xpack.monitoring.collection.enabled: true
xpack.monitoring.exporters.my_local.type: local
xpack.monitoring.exporters.my_local.use_ingest: false
xpack.security.authc.api_key.enabled: true
# https-es
xpack.security.enabled: true
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "/usr/local/es/config/certs/http.p12"
xpack.security.http.ssl.truststore.path: "/usr/local/es/config/certs/http.p12"
# https-kibana
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.client_authentication: required
xpack.security.transport.ssl.keystore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "/usr/local/es/config/certs/elastic-certificates.p12"
EOF
# 配置内置 JAVA
cat >> /usr/local/es/bin/elasticsearch << EOF
export JAVA_HOME=/usr/local/es/jdk
export PATH=$JAVA_HOME/bin:$PATH
EOF
#  调整运行内存
cat >> /usr/local/es/config/jvm.options << EOF
-Xms256m
-Xmx256m
EOF
chmod u+x /usr/local/es/bin
chown -R es:es /usr/local/es
#  端口
firewall-cmd --zone=public --add-port=9200/tcp --permanent;
firewall-cmd --zone=public --add-port=9300/tcp --permanent;
firewall-cmd --zone=public --add-service=http --permanent;
firewall-cmd --zone=public --add-service=https --permanent;
firewall-cmd --reload;firewall-cmd --list-all;
#  注册服务项
cat > /usr/lib/systemd/system/es.service << "EOF"
[Unit]
Description=elasticsearch
After=network.target

[Service]
Type=forking
User=es
ExecStart=/usr/local/es/bin/elasticsearch -d
PrivateTmp=true
# 指定此过程能够关上的最大文件数
LimitNOFILE=65535
# 指定此过程能够关上的最大过程数
LimitNPROC=65535
# 最大虚拟内存
LimitAS=infinity
# 最大文件大小
LimitFSIZE=infinity
# 超时设置 0- 永不超时
TimeoutStopSec=0
# SIGTERM 是进行 java 过程的信号
KillSignal=SIGTERM
# 信号只发送给给 JVM
KillMode=process
# java 过程不会被杀掉
SendSIGKILL=no
# 失常退出状态
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF
chmod 755 /usr/lib/systemd/system/es.service
systemctl enable es.service
systemctl start es.service
systemctl status es.service
}
install_es