Docker是一组平台即服务（PaaS）的产品。它基于操作系统层级的虚拟化技术，将软件与其依赖项打包为容器。托管容器的软件称为Docker引擎。docker能够帮助开发者在轻量级容器中自动部署应用程序，并使得不同容器中的应用程序彼此隔离，高效工作。该服务有免费和高级版本。它于2013年首次发布，由Docker, Inc. 开发
使用docker容器采用Raft模式部署 kafka，去掉kafka对zookeeper的依赖
Kafka是由Apache软件基金会开发的一个开源流处理平台，由Scala和Java编写。Kafka是一种高吞吐量的分布式发布订阅消息系统，它可以处理消费者在网站中的所有动作流数据。 这种动作（网页浏览，搜索和其他用户的行动）是在现代网络上的许多社会功能的一个关键因素。 这些数据通常是由于吞吐量的要求而通过处理日志和日志聚合来解决。 对于像Hadoop一样的日志数据和离线分析系统，但又要求实时处理的限制，这是一个可行的解决方案。Kafka的目的是通过Hadoop的并行加载机制来统一线上和离线的消息处理，也是为了通过集群来提供实时的消息。
docker compose 文件
services:
controller-1:
image: apache/kafka:3.9.0
container_name: controller-1
environment:
KAFKA_NODE_ID: 1
KAFKA_PROCESS_ROLES: controller
KAFKA_LISTENERS: CONTROLLER://controller-1:9093
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_KRAFT_METADATA_LOG_DIR: /var/lib/kafka/kraft-metadata
volumes:
- "../data/kafka/controller-1:/var/lib/kafka/kraft-metadata"
healthcheck:
test: ["CMD", "nc", "-z", "controller-1", "9093"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
controller-2:
image: apache/kafka:3.9.0
container_name: controller-2
environment:
KAFKA_NODE_ID: 2
KAFKA_PROCESS_ROLES: controller
KAFKA_LISTENERS: CONTROLLER://controller-2:9093
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_KRAFT_METADATA_LOG_DIR: /var/lib/kafka/kraft-metadata
volumes:
- "../data/kafka/controller-2:/var/lib/kafka/kraft-metadata"
healthcheck:
test: ["CMD", "nc", "-z", "controller-2", "9093"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
controller-3:
image: apache/kafka:3.9.0
container_name: controller-3
environment:
KAFKA_NODE_ID: 3
KAFKA_PROCESS_ROLES: controller
KAFKA_LISTENERS: CONTROLLER://controller-3:9093
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_KRAFT_METADATA_LOG_DIR: /var/lib/kafka/kraft-metadata
volumes:
- "../data/kafka/controller-3:/var/lib/kafka/kraft-metadata"
healthcheck:
test: ["CMD", "nc", "-z", "controller-3", "9093"]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
broker-1:
image: apache/kafka:3.9.0
container_name: broker-1
user: "0:0"
ports:
- 29092:9092
environment:
KAFKA_NODE_ID: 4
KAFKA_PROCESS_ROLES: broker
KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker-1:19092,PLAINTEXT_HOST://host.docker.internal:29092'
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_LOG_DIRS: /var/lib/kafka/data
KAFKA_BROKER_REGISTRATION_TIMEOUT_MS: 60000
KAFKA_BROKER_HEARTBEAT_INTERVAL_MS: 1000
depends_on:
controller-1:
condition: service_healthy
controller-2:
condition: service_healthy
controller-3:
condition: service_healthy
volumes:
- "../data/kafka/broker-1:/var/lib/kafka/data"
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "19092" ]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
broker-2:
image: apache/kafka:3.9.0
container_name: broker-2
user: "0:0"
ports:
- 39092:9092
environment:
KAFKA_NODE_ID: 5
KAFKA_PROCESS_ROLES: broker
KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker-2:19092,PLAINTEXT_HOST://host.docker.internal:39092'
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_LOG_DIRS: /var/lib/kafka/data
KAFKA_BROKER_REGISTRATION_TIMEOUT_MS: 60000
KAFKA_BROKER_HEARTBEAT_INTERVAL_MS: 1000
depends_on:
controller-1:
condition: service_healthy
controller-2:
condition: service_healthy
controller-3:
condition: service_healthy
volumes:
- "../data/kafka/broker-2:/var/lib/kafka/data"
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "19092" ]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
broker-3:
image: apache/kafka:3.9.0
container_name: broker-3
user: "0:0"
ports:
- 49092:9092
environment:
KAFKA_NODE_ID: 6
KAFKA_PROCESS_ROLES: broker
KAFKA_LISTENERS: 'PLAINTEXT://:19092,PLAINTEXT_HOST://:9092'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker-3:19092,PLAINTEXT_HOST://host.docker.internal:49092'
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@controller-1:9093,2@controller-2:9093,3@controller-3:9093
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_LOG_DIRS: /var/lib/kafka/data
KAFKA_BROKER_REGISTRATION_TIMEOUT_MS: 60000
KAFKA_BROKER_HEARTBEAT_INTERVAL_MS: 1000
depends_on:
controller-1:
condition: service_healthy
controller-2:
condition: service_healthy
controller-3:
condition: service_healthy
volumes:
- "../data/kafka/broker-3:/var/lib/kafka/data"
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "19092" ]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
networks:
- kafka-network
kafka-ui:
image: provectuslabs/kafka-ui:latest
container_name: kafka-ui
ports:
- 8080:8080
environment:
KAFKA_CLUSTERS_0_NAME: kafka-cluster-1
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker-1:19092,broker-2:19092,broker-3:19092
TZ: Asia/Shanghai
LANG: C.UTF-8
depends_on:
broker-1:
condition: service_healthy
broker-2:
condition: service_healthy
broker-3:
condition: service_healthy
networks:
- kafka-network
networks:
kafka-network:
driver: bridge验证测试
创建Topic
docker exec -it broker-1 /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 --create --topic test-topic --partitions 3 --replication-factor 3查看Topic
docker exec -it broker-1 /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 --listKafka UI
快速部署
docker run -p 8080 \
--name kafka-ui \
-e KAFKA_CLUSTERS_0_NAME=kafka9092 \
-e KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=localhost:9092 \
-e KAFKA_CLUSTERS_1_NAME=kafka9093 \
-e KAFKA_CLUSTERS_1_BOOTSTRAPSERVERS=localhost:9093 \
-e TZ=Asia/Shanghai \
-e LANG=C.UTF-8 \
-d provectuslabs/kafka-ui:latest 增加登录项
默认kafka-ui是免登录的，为了防止所有人都可以登录产生误操作问题，在docker部署的时候可以添加如下四个环境变量配置用户的登录密码。用户名密码根据自己需求修改倒数两行中的admin。
-e SERVER_SERVLET_CONTEXT_PATH="/" \
-e AUTH_TYPE="LOGIN_FORM" \
-e SPRING_SECURITY_USER_NAME=admin \
-e SPRING_SECURITY_USER_PASSWORD="admin" \优点
1.topic管理：支持查看topic元信息， 增加、删除topic，清除topic信息，往topic中添加数据等。
2.docker部署，非常方便
3.功能齐全，该有的功能全都有
4.集群连接方便，不管是连接旧版本的kafka还是3.3.1及以后版本的kafka，只需要指定kafka的ip端口即可，不需要指定任何其他内容。
缺点
1.不支持页面添加集群,官方也解释了，这是一个无状态的服务，所以情有可原。但是因为项目是Springboot写的，docker部署的，如果添加集群可以重建docker容器，或者修改配置文件，重启容器，具体可以查看项目github说明，因为我没有用配置文件，这里没有详细介绍。
连接带认证的kafka集群
连接带认证的kafka集群在kafka-ui的github上面也有，但是文章写的不太全面，我测试了如下两种带认证集群的连接方式。需要可以参考。
连接SASL_PLAINTEXT认证的Kafka
如果集群没有配置SASL_PLAINTEXT认证可以参考我写的这篇文章。
连接SASL_PLAINTEXT认证的kafka需要添加如下三个环境变量，KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG变量指定的用户名密码需要与实际情况一致。
docker run -p 1992:8080 \
--name kafka-ui \
-e KAFKA_CLUSTERS_0_NAME=kafka9092 \
-e KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=localhost:9092 \
-e KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL=SASL_PLAINTEXT \
-e KAFKA_CLUSTERS_0_PROPERTIES_SASL_MECHANISM=PLAIN \
-e KAFKA_CLUSTERS_0_PROPERTIES_SASL_JAAS_CONFIG='org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin";' \
-e TZ=Asia/Shanghai \
-e LANG=C.UTF-8 \
-d provectuslabs/kafka-ui:latest5.官方文档非常详细，需要查看详细说明，可以点击链接跳转。