【软件部署】用docker部署Apache Kafka 集群架构isolated模式带SSL

说明

带SSL，增强安全

配置

创建文件compose.yaml

# 有SSL

services:
init-kafka-perms:
image: busybox:latest
command: sh -c "chown -R 1000:1000 /controller-1 /controller-2 /controller-3 /kafka1 /kafka2 /kafka3"
volumes:
– controller-1:/controller-1
– controller-2:/controller-2
– controller-3:/controller-3
– kafka1-logs:/kafka1
– kafka2-logs:/kafka2
– kafka3-logs:/kafka3
restart: "no"
container_name: kafka-perms-fix
networks:
– kafka

controller-1:
image: apache/kafka:4.2.0
hostname: controller-1
container_name: kafka-controller-1
restart: unless-stopped
environment:
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: 'controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:29092'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
depends_on:
init-kafka-perms:
condition: service_completed_successfully
networks:
– kafka
volumes:
– controller-1:/tmp/kraft-combined-logs
healthcheck:
test: nc -z localhost 29092 || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 10s

controller-2:
image: apache/kafka:4.2.0
hostname: controller-2
container_name: kafka-controller-2
restart: unless-stopped
environment:
KAFKA_NODE_ID: 2
KAFKA_PROCESS_ROLES: 'controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:29092'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
depends_on:
init-kafka-perms:
condition: service_completed_successfully
networks:
– kafka
volumes:
– controller-2:/tmp/kraft-combined-logs
healthcheck:
test: nc -z localhost 29092 || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 10s

controller-3:
image: apache/kafka:4.2.0
hostname: controller-3
container_name: kafka-controller-3
restart: unless-stopped
environment:
KAFKA_NODE_ID: 3
KAFKA_PROCESS_ROLES: 'controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:29092'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
depends_on:
init-kafka-perms:
condition: service_completed_successfully
networks:
– kafka
volumes:
– controller-3:/tmp/kraft-combined-logs
healthcheck:
test: nc -z localhost 29092 || exit 1
interval: 30s
timeout: 5s
retries: 3
start_period: 10s

kafka-1:
image: apache/kafka:4.2.0
hostname: kafka-1
container_name: kafka-1
ports:
– 29093:9093
volumes:
– ./secrets:/etc/kafka/secrets
– kafka1-logs:/tmp/kraft-combined-logs
environment:
KAFKA_NODE_ID: 4
KAFKA_PROCESS_ROLES: 'broker'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
KAFKA_SECURITY_PROTOCOL: SSL
KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-1:19093,SSL://localhost:29093
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
KAFKA_SSL_CLIENT_AUTH: 'required'
KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
depends_on:
controller-1: { condition: service_healthy }
controller-2: { condition: service_healthy }
controller-3: { condition: service_healthy }
restart: unless-stopped
networks:
– kafka
healthcheck:
test: nc -z localhost 9093 || exit 1
interval: 60s
timeout: 5s
retries: 2
start_period: 30s

kafka-2:
image: apache/kafka:4.2.0
hostname: kafka-2
container_name: kafka-2
ports:
– 39093:9093
volumes:
– ./secrets:/etc/kafka/secrets
– kafka2-logs:/tmp/kraft-combined-logs
environment:
KAFKA_NODE_ID: 5
KAFKA_PROCESS_ROLES: 'broker'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
KAFKA_SECURITY_PROTOCOL: SSL
KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-2:19093,SSL://localhost:39093
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
KAFKA_SSL_CLIENT_AUTH: 'required'
KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
depends_on:
controller-1: { condition: service_healthy }
controller-2: { condition: service_healthy }
controller-3: { condition: service_healthy }
restart: unless-stopped
networks:
– kafka
healthcheck:
test: nc -z localhost 9093 || exit 1
interval: 60s
timeout: 5s
retries: 2
start_period: 30s

kafka-3:
image: apache/kafka:4.2.0
hostname: kafka-3
container_name: kafka-3
ports:
– 49093:9093
volumes:
– ./secrets:/etc/kafka/secrets
– kafka3-logs:/tmp/kraft-combined-logs
environment:
KAFKA_NODE_ID: 6
KAFKA_PROCESS_ROLES: 'broker'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
KAFKA_SECURITY_PROTOCOL: SSL
KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-3:19093,SSL://localhost:49093
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
CLUSTER_ID: '4L6g3nShT-eMCtK–X86sw'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
KAFKA_SSL_CLIENT_AUTH: 'required'
KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
depends_on:
controller-1: { condition: service_healthy }
controller-2: { condition: service_healthy }
controller-3: { condition: service_healthy }
restart: unless-stopped
networks:
– kafka
healthcheck:
test: nc -z localhost 9093 || exit 1
interval: 60s
timeout: 5s
retries: 2
start_period: 30s

volumes:
controller-1:
name: kafka-controller-1
controller-2:
name: kafka-controller-2
controller-3:
name: kafka-controller-3
kafka1-logs:
name: kafka1-logs
kafka2-logs:
name: kafka2-logs
kafka3-logs:
name: kafka3-logs

networks:
kafka:
name: kafka

创建目录secrets
自己生成文件或下载SSL模板文件https://github.com/apache/kafka/tree/trunk/docker/examples/fixtures/secrets

部署

docker compose up -d

测试
确认容器健康
功能测试待更新。