diff --git a/script/docker/elk/docker-compose.yml b/script/docker/elk/docker-compose.yml new file mode 100644 index 00000000..48abbbee --- /dev/null +++ b/script/docker/elk/docker-compose.yml @@ -0,0 +1,114 @@ +version: "3" +services: + es-master: + container_name: es-master + hostname: es-master + image: elasticsearch:7.1.1 + restart: always + ports: + - 9200:9200 + - 9300:9300 + volumes: + - ./elasticsearch/master/conf/es-master.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - ./elasticsearch/master/data:/usr/share/elasticsearch/data + - ./elasticsearch/master/logs:/usr/share/elasticsearch/logs + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + es-slave1: + container_name: es-slave1 + image: elasticsearch:7.1.1 + restart: always + ports: + - 9201:9200 + - 9301:9300 + volumes: + - ./elasticsearch/slave1/conf/es-slave1.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - ./elasticsearch/slave1/data:/usr/share/elasticsearch/data + - ./elasticsearch/slave1/logs:/usr/share/elasticsearch/logs + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + es-slave2: + container_name: es-slave2 + image: elasticsearch:7.1.1 + restart: always + ports: + - 9202:9200 + - 9302:9300 + volumes: + - ./elasticsearch/slave2/conf/es-slave2.yml:/usr/share/elasticsearch/config/elasticsearch.yml + - ./elasticsearch/slave2/data:/usr/share/elasticsearch/data + - ./elasticsearch/slave2/logs:/usr/share/elasticsearch/logs + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + es-head: + container_name: es-head + image: mobz/elasticsearch-head:5 + restart: always + ports: + - 9100:9100 + depends_on: + - es-master + - es-slave1 + - es-slave2 + + kibana: + container_name: kibana + hostname: kibana + image: kibana:7.1.1 + restart: always + ports: + - 5601:5601 + volumes: + - ./kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml + environment: + - elasticsearch.hosts=http://es-master:9200 + depends_on: + - es-master + - es-slave1 + - es-slave2 + + filebeat: + # 容器名称 + container_name: filebeat + # 主机名称 + hostname: filebeat + # 镜像 + image: docker.elastic.co/beats/filebeat:7.1.1 + # 重启机制 + restart: always + # 持久化挂载 + volumes: + - ./filebeat/conf/filebeat.yml:/usr/share/filebeat/filebeat.yml + # 映射到容器中[作为数据源] + - ./logs:/home/project/spring-boot-elasticsearch/logs + - ./filebeat/logs:/usr/share/filebeat/logs + - ./filebeat/data:/usr/share/filebeat/data + # 将指定容器连接到当前连接,可以设置别名,避免ip方式导致的容器重启动态改变的无法连接情况 + links: + - logstash + # 依赖服务[可无] + depends_on: + - es-master + - es-slave1 + - es-slave2 + + logstash: + container_name: logstash + hostname: logstash + image: logstash:7.1.1 + command: logstash -f ./conf/logstash-filebeat.conf + restart: always + volumes: + # 映射到容器中 + - ./logstash/conf/logstash-filebeat.conf:/usr/share/logstash/conf/logstash-filebeat.conf + environment: + - elasticsearch.hosts=http://es-master:9200 + ports: + - 5044:5044 + depends_on: + - es-master + - es-slave1 + - es-slave2 diff --git a/script/docker/elk/elk-deploy.sh b/script/docker/elk/elk-deploy.sh new file mode 100644 index 00000000..de5786e9 --- /dev/null +++ b/script/docker/elk/elk-deploy.sh @@ -0,0 +1,87 @@ +#./bin/bash +# 定义颜色 +BLUE_COLOR="\033[36m" +RED_COLOR="\033[31m" +GREEN_COLOR="\033[32m" +VIOLET_COLOR="\033[35m" +RES="\033[0m" + +echo -e "${BLUE_COLOR}# ######################################################################${RES}" +echo -e "${BLUE_COLOR}# Docker ELK Deploy Script #${RES}" +echo -e "${BLUE_COLOR}# ######################################################################${RES}" + +# 创建目录 +echo -e "${BLUE_COLOR}---> create [elasticsearch]directory start.${RES}" +if [ ! -d "./elasticsearch/" ]; then +mkdir -p ./elasticsearch/master/conf ./elasticsearch/master/data ./elasticsearch/master/logs \ + ./elasticsearch/slave1/conf ./elasticsearch/slave1/data ./elasticsearch/slave1/logs \ + ./elasticsearch/slave2/conf ./elasticsearch/slave2/data ./elasticsearch/slave2/logs +fi + +echo -e "${RED_COLOR}---> create [kibana]directory start.${RES}" +if [ ! -d "./kibana/" ]; then +mkdir -p ./kibana/conf ./kibana/logs +fi + + echo -e "${GREEN_COLOR}---> create [logstash]directory start.${RES}" + if [ ! -d "./logstash/" ]; then + mkdir -p ./logstash/conf ./logstash/logs + fi + +echo -e "${GREEN_COLOR}---> create [filebeat]directory start.${RES}" +if [ ! -d "./filebeat/" ]; then +mkdir -p ./filebeat/conf ./filebeat/logs ./filebeat/data +fi + +echo -e "${VIOLET_COLOR}---> create [nginx]directory start.${RES}" +if [ ! -d "./nginx/" ]; then +mkdir -p ./nginx/conf ./nginx/logs ./nginx/www +fi +echo -e "${BLUE_COLOR}===> create directory success.${RES}" + +# 目录授权(data/logs 都要授读/写权限) +echo -e "${BLUE_COLOR}---> directory authorize start.${RES}" +if [ -d "./elasticsearch/" ]; then +chmod 777 ./elasticsearch/master/data/ ./elasticsearch/master/logs/ \ + ./elasticsearch/slave1/data/ ./elasticsearch/slave1/logs/ \ + ./elasticsearch/slave2/data/ ./elasticsearch/slave2/logs +fi + +if [ -d "./filebeat/" ]; then +chmod 777 ./filebeat/data/ ./filebeat/logs/ +fi +echo -e "${BLUE_COLOR}===> directory authorize success.${RES}" + +# 移动配置文件 +echo -e "${BLUE_COLOR}---> move [elasticsearch]config file start.${RES}" +if [ -f "./es-master.yml" ] && [ -f "./es-slave1.yml" ] && [ -f "./es-slave2.yml" ]; then +mv ./es-master.yml ./elasticsearch/master/conf +mv ./es-slave1.yml ./elasticsearch/slave1/conf +mv ./es-slave2.yml ./elasticsearch/slave2/conf +fi + +echo -e "${RED_COLOR}---> move [kibana]config file start.${RES}" +if [ -f "./kibana.yml" ]; then +mv ./kibana.yml ./kibana/conf +fi + + echo -e "${GREEN_COLOR}---> move [logstash]config file start.${RES}" + if [ -f "./logstash-filebeat.conf" ]; then + mv ./logstash-filebeat.conf ./logstash/conf + fi + +echo -e "${GREEN_COLOR}---> move [filebeat]config file start.${RES}" +if [ -f "./filebeat.yml" ]; then +mv ./filebeat.yml ./filebeat/conf +fi + +echo -e "${VIOLET_COLOR}---> move [nginx]config file start.${RES}" +if [ -f "./nginx.conf" ]; then +mv ./nginx.conf ./nginx/conf +fi +echo -e "${BLUE_COLOR}===> move config files success.${RES}" +echo -e "${GREEN_COLOR}>>>>>>>>>>>>>>>>>> The End <<<<<<<<<<<<<<<<<<${RES}" + +# 部署项目 +echo -e "${BLUE_COLOR}==================> Docker deploy Start <==================${RES}" +docker-compose up --build -d diff --git a/script/docker/elk/elk-undeploy.sh b/script/docker/elk/elk-undeploy.sh new file mode 100644 index 00000000..9e079024 --- /dev/null +++ b/script/docker/elk/elk-undeploy.sh @@ -0,0 +1,16 @@ +#./bin/bash +# 定义颜色 +BLUE_COLOR="\033[36m" +RED_COLOR="\033[31m" +GREEN_COLOR="\033[32m" +VIOLET_COLOR="\033[35m" +RES="\033[0m" + +echo -e "${BLUE_COLOR}# ######################################################################${RES}" +echo -e "${BLUE_COLOR}# Docker ELK UnDeploy Script #${RES}" +echo -e "${BLUE_COLOR}# ######################################################################${RES}" + +# 部署项目 +echo -e "${BLUE_COLOR}==================> Docker undeploy Start <==================${RES}" +docker-compose stop +docker-compose rm diff --git a/script/docker/elk/es-master.yml b/script/docker/elk/es-master.yml new file mode 100644 index 00000000..8d7c2982 --- /dev/null +++ b/script/docker/elk/es-master.yml @@ -0,0 +1,28 @@ +# 集群名称 +cluster.name: es-cluster +# 节点名称 +node.name: es-master +# 是否可以成为master节点 +node.master: true +# 是否允许该节点存储数据,默认开启 +node.data: false +# 网络绑定 +network.host: 0.0.0.0 +# 设置对外服务的http端口 +http.port: 9200 +# 设置节点间交互的tcp端口 +transport.port: 9300 +# 集群发现 +discovery.seed_hosts: + - es-master + - es-slave1 + - es-slave2 +# 手动指定可以成为 mater 的所有节点的 name 或者 ip,这些配置将会在第一次选举中进行计算 +cluster.initial_master_nodes: + - es-master +# 支持跨域访问 +http.cors.enabled: true +http.cors.allow-origin: "*" +# 安全认证 +xpack.security.enabled: false +#http.cors.allow-headers: "Authorization" diff --git a/script/docker/elk/es-slave1.yml b/script/docker/elk/es-slave1.yml new file mode 100644 index 00000000..b252387b --- /dev/null +++ b/script/docker/elk/es-slave1.yml @@ -0,0 +1,28 @@ +# 集群名称 +cluster.name: es-cluster +# 节点名称 +node.name: es-slave1 +# 是否可以成为master节点 +node.master: true +# 是否允许该节点存储数据,默认开启 +node.data: true +# 网络绑定 +network.host: 0.0.0.0 +# 设置对外服务的http端口 +http.port: 9201 +# 设置节点间交互的tcp端口 +#transport.port: 9301 +# 集群发现 +discovery.seed_hosts: + - es-master + - es-slave1 + - es-slave2 +# 手动指定可以成为 mater 的所有节点的 name 或者 ip,这些配置将会在第一次选举中进行计算 +cluster.initial_master_nodes: + - es-master +# 支持跨域访问 +http.cors.enabled: true +http.cors.allow-origin: "*" +# 安全认证 +xpack.security.enabled: false +#http.cors.allow-headers: "Authorization" diff --git a/script/docker/elk/es-slave2.yml b/script/docker/elk/es-slave2.yml new file mode 100644 index 00000000..8f489c2d --- /dev/null +++ b/script/docker/elk/es-slave2.yml @@ -0,0 +1,28 @@ +# 集群名称 +cluster.name: es-cluster +# 节点名称 +node.name: es-slave2 +# 是否可以成为master节点 +node.master: true +# 是否允许该节点存储数据,默认开启 +node.data: true +# 网络绑定 +network.host: 0.0.0.0 +# 设置对外服务的http端口 +http.port: 9202 +# 设置节点间交互的tcp端口 +#transport.port: 9302 +# 集群发现 +discovery.seed_hosts: + - es-master + - es-slave1 + - es-slave2 +# 手动指定可以成为 mater 的所有节点的 name 或者 ip,这些配置将会在第一次选举中进行计算 +cluster.initial_master_nodes: + - es-master +# 支持跨域访问 +http.cors.enabled: true +http.cors.allow-origin: "*" +# 安全认证 +xpack.security.enabled: false +#http.cors.allow-headers: "Authorization" diff --git a/script/docker/elk/filebeat.yml b/script/docker/elk/filebeat.yml new file mode 100644 index 00000000..a7d2b37c --- /dev/null +++ b/script/docker/elk/filebeat.yml @@ -0,0 +1,33 @@ +filebeat.inputs: + - type: log + enabled: true + paths: + # 当前目录下的所有.log文件 + - /home/project/spring-boot-elasticsearch/logs/*.log + multiline.pattern: ^\[ + multiline.negate: true + multiline.match: after + +filebeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +setup.template.settings: + index.number_of_shards: 1 + +setup.dashboards.enabled: false + +setup.kibana: + host: "http://kibana:5601" + +# 不直接传输至ES +#output.elasticsearch: +# hosts: ["http://es-master:9200"] +# index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}" + +output.logstash: + hosts: ["logstash:5044"] + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ diff --git a/script/docker/elk/kibana.yml b/script/docker/elk/kibana.yml new file mode 100644 index 00000000..80c2d288 --- /dev/null +++ b/script/docker/elk/kibana.yml @@ -0,0 +1,8 @@ +# 服务端口 +server.port: 5601 +# 服务IP +server.host: "0.0.0.0" +# ES +elasticsearch.hosts: ["http://es-master:9200"] +# 汉化 +i18n.locale: "zh-CN" diff --git a/script/docker/elk/logstash-filebeat.conf b/script/docker/elk/logstash-filebeat.conf new file mode 100644 index 00000000..3c2938fa --- /dev/null +++ b/script/docker/elk/logstash-filebeat.conf @@ -0,0 +1,23 @@ +input { + # 来源beats + beats { + # 端口 + port => "5044" + } +} +# 分析、过滤插件,可以多个 +filter { + grok { + match => { "message" => "%{COMBINEDAPACHELOG}"} + } + geoip { + source => "clientip" + } +} +output { + # 选择elasticsearch + elasticsearch { + hosts => ["http://es-master:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" + } +}