ELK + Redis Docker 企业级部署落地方案

ELK + Redis Docker 企业级部署落地方案

📋 方案概述

架构设计

🚀 一、准备工作

1.1 环境要求

# 最低硬件配置
– 内存: 16GB RAM (推荐32GB+)
– 存储: 200GB SSD (推荐NVMe)
– CPU: 8核 (推荐16核+)
– 网络: 千兆网卡

# 软件要求
– Docker 20.10+
– Docker Compose 2.0+
– 操作系统: Ubuntu 20.04/22.04, CentOS 8+

# 端口规划
┌─────────────┬─────────┬───────────────┐
│ 服务 │ 端口 │ 用途 │
├─────────────┼─────────┼───────────────┤
│ Elasticsearch│ 9200 │ HTTP API │
│ Elasticsearch│ 9300 │ 节点通信 │
│ Kibana │ 5601 │ Web界面 │
│ Logstash │ 5044 │ Beats输入 │
│ Logstash │ 5000 │ TCP输入 │
│ Redis │ 6379 │ 缓存队列 │
│ Cerebro │ 9000 │ ES集群管理 │
│ Prometheus │ 9090 │ 监控 │
│ Grafana │ 3000 │ 可视化 │
└─────────────┴─────────┴───────────────┘

1.2 目录结构

elk-docker-enterprise/
├── .env # 环境变量配置
├── docker-compose.yml # 主编排文件
├── README.md # 部署文档
├── LICENSE
├── scripts/ # 维护脚本
│ ├── init.sh
│ ├── backup.sh
│ ├── restore.sh
│ └── health-check.sh
├── configs/ # 配置文件
│ ├── elasticsearch/
│ │ ├── elasticsearch.yml
│ │ ├── jvm.options
│ │ └── log4j2.properties
│ ├── kibana/
│ │ └── kibana.yml
│ ├── logstash/
│ │ ├── logstash.yml
│ │ ├── pipelines.yml
│ │ ├── patterns/ # Grok模式
│ │ └── conf.d/ # 管道配置
│ ├── filebeat/
│ │ └── filebeat.yml
│ └── redis/
│ └── redis.conf
├── data/ # 持久化数据
│ ├── elasticsearch/
│ ├── redis/
│ ├── logstash/
│ └── prometheus/
├── logs/ # 容器日志
├── certificates/ # TLS证书
│ ├── ca.crt
│ ├── elasticsearch.crt
│ └── elasticsearch.key
└── grafana/
├── dashboards/
└── provisioning/

1.3 创建部署目录

# 创建项目目录结构
mkdir -p elk-docker-enterprise/{configs,data,logs,scripts,certificates,grafana}
cd elk-docker-enterprise

# 创建子目录
mkdir -p configs/{elasticsearch,kibana,logstash/{patterns,conf.d},filebeat,redis}
mkdir -p data/{elasticsearch,redis,logstash,prometheus}
mkdir -p grafana/{dashboards,provisioning}
mkdir -p certificates
mkdir -p logs/{elasticsearch,kibana,logstash,filebeat,redis}

🐳 二、Docker Compose 编排文件

2.1 环境变量文件 (.env)

# ELK 版本配置
ELK_VERSION=8.11.0
REDIS_VERSION=7.2

# 网络配置
NETWORK_NAME=elk-network
SUBNET=172.20.0.0/16

# Elasticsearch 配置
ES_CLUSTER_NAME=elk-enterprise-cluster
ES_NODE_NAME=node-1
ES_DISCOVERY_TYPE=single-node
ES_HEAP_SIZE=4g
ES_JAVA_OPTS=-Xms4g -Xmx4g

# 安全配置
ELASTIC_PASSWORD=YourSecurePassword123!
KIBANA_PASSWORD=KibanaSecurePass456!
LOGSTASH_SYSTEM_PASSWORD=LogstashSystem789!
BEATS_SYSTEM_PASSWORD=BeatsSystemPass321!

# Redis 配置
REDIS_PASSWORD=RedisSecurePass654!
REDIS_MAXMEMORY=2gb

# 存储路径
ELK_DATA_PATH=/data/elk
ELK_LOGS_PATH=/var/log/elk

# 资源限制
ES_CPU_LIMIT=4
ES_MEMORY_LIMIT=8g
LOGSTASH_CPU_LIMIT=2
LOGSTASH_MEMORY_LIMIT=4g
KIBANA_CPU_LIMIT=1
KIBANA_MEMORY_LIMIT=2g

# 节点数量
ES_NODES_COUNT=3
LOGSTASH_NODES_COUNT=2
KIBANA_NODES_COUNT=2

2.2 Docker Compose 主文件 (docker-compose.yml)

version: \’3.8\’

networks:
elk-network:
driver: bridge
ipam:
config:
– subnet: ${SUBNET:-172.20.0.0/16}
attachable: true

volumes:
elasticsearch-data:
driver: local
driver_opts:
type: none
o: bind
device: ${ELK_DATA_PATH:-./data}/elasticsearch
redis-data:
driver: local
driver_opts:
type: none
o: bind
device: ${ELK_DATA_PATH:-./data}/redis
prometheus-data:
driver: local

services:
# ============== Redis 集群（缓冲队列）==============
redis-master:
image: redis:${REDIS_VERSION:-7.2}-alpine
container_name: redis-master
hostname: redis-master
restart: unless-stopped
command: redis-server /usr/local/etc/redis/redis.conf –requirepass ${REDIS_PASSWORD}
environment:
– REDIS_REPLICATION_MODE=master
volumes:
– ./configs/redis/redis.conf:/usr/local/etc/redis/redis.conf
– redis-data:/data
– ./logs/redis:/var/log/redis
ports:
– \”6379:6379\”
networks:
elk-network:
ipv4_address: 172.20.0.10
healthcheck:
test: [\”CMD\”, \”redis-cli\”, \”-a\”, \”${REDIS_PASSWORD}\”, \”ping\”]
interval: 10s
timeout: 5s
retries: 3
deploy:
resources:
limits:
cpus: \’1\’
memory: 2G
reservations:
cpus: \’0.5\’
memory: 1G

redis-replica:
image: redis:${REDIS_VERSION:-7.2}-alpine
container_name: redis-replica
hostname: redis-replica
restart: unless-stopped
command: redis-server /usr/local/etc/redis/redis.conf –requirepass ${REDIS_PASSWORD} –slaveof redis-master 6379
environment:
– REDIS_REPLICATION_MODE=slave
– REDIS_MASTER_HOST=redis-master
– REDIS_MASTER_PORT_NUMBER=6379
– REDIS_MASTER_PASSWORD=${REDIS_PASSWORD}
volumes:
– ./configs/redis/redis.conf:/usr/local/etc/redis/redis.conf
depends_on:
– redis-master
networks:
elk-network:
ipv4_address: 172.20.0.11
healthcheck:
test: [\”CMD\”, \”redis-cli\”, \”-a\”, \”${REDIS_PASSWORD}\”, \”ping\”]
interval: 10s
timeout: 5s
retries: 3

# ============== Elasticsearch 集群 ==============
elasticsearch-node1:
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
container_name: elasticsearch-node1
hostname: elasticsearch-node1
restart: unless-stopped
environment:
– node.name=elasticsearch-node1
– cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
– cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
– discovery.seed_hosts=elasticsearch-node2,elasticsearch-node3
– bootstrap.memory_lock=true
– \”ES_JAVA_OPTS=${ES_JAVA_OPTS:–Xms4g -Xmx4g}\”
– xpack.security.enabled=true
– xpack.security.http.ssl.enabled=false
– xpack.security.transport.ssl.enabled=false
– xpack.license.self_generated.type=basic
– ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
– LOGSTASH_SYSTEM_PASSWORD=${LOGSTASH_SYSTEM_PASSWORD}
– KIBANA_SYSTEM_PASSWORD=${KIBANA_PASSWORD}
– BEATS_SYSTEM_PASSWORD=${BEATS_SYSTEM_PASSWORD}
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
volumes:
– elasticsearch-data:/usr/share/elasticsearch/data
– ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
– ./configs/elasticsearch/jvm.options:/usr/share/elasticsearch/config/jvm.options
– ./logs/elasticsearch:/usr/share/elasticsearch/logs
ports:
– \”9200:9200\”
– \”9300:9300\”
networks:
elk-network:
ipv4_address: 172.20.0.20
healthcheck:
test: [\”CMD-SHELL\”, \”curl -s -u elastic:${ELASTIC_PASSWORD} http://localhost:9200/_cluster/health | grep -q \’\\\”status\\\”:\\\”green\\\”\’\”]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
deploy:
resources:
limits:
cpus: \’${ES_CPU_LIMIT:-4}\’
memory: ${ES_MEMORY_LIMIT:-8g}
reservations:
cpus: \’${ES_CPU_LIMIT:-2}\’
memory: ${ES_MEMORY_LIMIT:-4g}

elasticsearch-node2:
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
container_name: elasticsearch-node2
hostname: elasticsearch-node2
restart: unless-stopped
environment:
– node.name=elasticsearch-node2
– cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
– cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
– discovery.seed_hosts=elasticsearch-node1,elasticsearch-node3
– bootstrap.memory_lock=true
– \”ES_JAVA_OPTS=${ES_JAVA_OPTS:–Xms4g -Xmx4g}\”
– xpack.security.enabled=true
– xpack.security.http.ssl.enabled=false
– xpack.security.transport.ssl.enabled=false
– ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
ulimits:
memlock:
soft: -1
hard: -1
volumes:
– elasticsearch-data-node2:/usr/share/elasticsearch/data
– ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
– ./logs/elasticsearch-node2:/usr/share/elasticsearch/logs
networks:
elk-network:
ipv4_address: 172.20.0.21
depends_on:
– elasticsearch-node1
healthcheck:
test: [\”CMD-SHELL\”, \”curl -s http://localhost:9200/_cluster/health | grep -q \’\\\”status\\\”:\\\”green\\\”\’\”]
interval: 30s
timeout: 10s
retries: 3

elasticsearch-node3:
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
container_name: elasticsearch-node3
hostname: elasticsearch-node3
restart: unless-stopped
environment:
– node.name=elasticsearch-node3
– cluster.name=${ES_CLUSTER_NAME:-elk-enterprise-cluster}
– cluster.initial_master_nodes=elasticsearch-node1,elasticsearch-node2,elasticsearch-node3
– discovery.seed_hosts=elasticsearch-node1,elasticsearch-node2
– bootstrap.memory_lock=true
– \”ES_JAVA_OPTS=${ES_JAVA_OPTS:–Xms4g -Xmx4g}\”
– xpack.security.enabled=true
– xpack.security.http.ssl.enabled=false
– xpack.security.transport.ssl.enabled=false
– ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
ulimits:
memlock:
soft: -1
hard: -1
volumes:
– elasticsearch-data-node3:/usr/share/elasticsearch/data
– ./configs/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
– ./logs/elasticsearch-node3:/usr/share/elasticsearch/logs
networks:
elk-network:
ipv4_address: 172.20.0.22
depends_on:
– elasticsearch-node1
healthcheck:
test: [\”CMD-SHELL\”, \”curl -s http://localhost:9200/_cluster/health | grep -q \’\\\”status\\\”:\\\”green\\\”\’\”]
interval: 30s
timeout: 10s
retries: 3

# ============== Logstash 集群 ==============
logstash-node1:
image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
container_name: logstash-node1
hostname: logstash-node1
restart: unless-stopped
environment:
– LS_JAVA_OPTS=-Xmx2g -Xms2g
– XPACK_MONITORING_ENABLED=true
– XPACK_MONITORING_ELASTICSEARCH_URL=http://elasticsearch-node1:9200
– XPACK_MONITORING_ELASTICSEARCH_USERNAME=logstash_system
– XPACK_MONITORING_ELASTICSEARCH_PASSWORD=${LOGSTASH_SYSTEM_PASSWORD}
– ELASTICSEARCH_HOSTS=http://elasticsearch-node1:9200
volumes:
– ./configs/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml
– ./configs/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml
– ./configs/logstash/conf.d:/usr/share/logstash/pipeline
– ./configs/logstash/patterns:/usr/share/logstash/patterns
– ./logs/logstash:/usr/share/logstash/logs
ports:
– \”5044:5044\” # Beats input
– \”5000:5000\” # TCP input
– \”5001:5001\” # UDP input
– \”9600:9600\” # Logstash monitoring API
networks:
elk-network:
ipv4_address: 172.20.0.30
depends_on:
– elasticsearch-node1
– redis-master
healthcheck:
test: [\”CMD\”, \”curl\”, \”-f\”, \”http://localhost:9600\”]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
deploy:
resources:
limits:
cpus: \’${LOGSTASH_CPU_LIMIT:-2}\’
memory: ${LOGSTASH_MEMORY_LIMIT:-4g}
reservations:
cpus: \’1\’
memory: 2g

logstash-node2:
image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
container_name: logstash-node2
hostname: logstash-node2
restart: unless-stopped
environment:
– LS_JAVA_OPTS=-Xmx2g -Xms2g
– XPACK_MONITORING_ENABLED=true
– XPACK_MONITORING_ELASTICSEARCH_URL=http://