[Kafka] Kafka Cluster 구축을 위한 docker-compose 파일

- 목차

 

들어가며.

이번 글은 Kafka 와 관련한 여러 테스트를 손쉽게 진행하기 위해서 Kafka 관련 Docker Compose File 들을 기록합니다.

 

 

JdbcSourceConnector 실행을 위한 Cluster 구성.

1. 1개의 Kakfa Broker
2. 1개의 Zookeeper
3. 1개의 Kafka Connect Worker
4. Kafka UI

로 구성된 Docker Compose Yaml 파일입니다.

 

cat <<'EOF'> /tmp/docker-compose.yaml
version: '3.8'

services:
  zookeeper1:
    image: confluentinc/cp-zookeeper:7.8.0
    hostname: zookeeper1
    container_name: zookeeper1
    networks:
      - kafka
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
    restart: always

  kafka1:
    image: confluentinc/cp-kafka:7.8.0
    hostname: kafka1
    container_name: kafka1
    networks:
      - kafka
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
    restart: always

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    container_name: kafka-ui
    networks:
      - kafka
    environment:
      DYNAMIC_CONFIG_ENABLED: 'true'
      KAFKA_CLUSTERS_0_NAME: test1
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka1:9092
    ports:
      - 8080:8080
    restart: always

  connect-1:
    image: westlife0615/kafka-jdbc-connect:latest
    hostname: connect-1
    container_name: connect-1
    networks:
      - kafka
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka1:9092
      CONNECT_REST_ADVERTISED_HOST_NAME: connect-1
      CONNECT_GROUP_ID: connect-cluster
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
    restart: always

networks:
  kafka:
    driver: bridge
    external: true
EOF


docker-compose -f /tmp/docker-compose.yaml -p kafka-connect up -d

 

위 docker-compose 가 실행되면, 아래와 같은 구성을 이룹니다.

 

 

 

Debezium Connector 실행을 위한 Cluster 구성.

1. 1개의 Kakfa Broker
2. 1개의 Zookeeper
3. 1개의 Kafka Connect Worker
4. Kafka UI

로 구성된 Docker Compose Yaml 파일입니다.

 

cat <<'EOF'> /tmp/docker-compose.yaml
version: '3.8'

services:
  zookeeper1:
    image: confluentinc/cp-zookeeper:7.8.0
    hostname: zookeeper1
    container_name: zookeeper1
    networks:
      - kafka
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
    restart: always

  kafka1:
    image: confluentinc/cp-kafka:7.8.0
    hostname: kafka1
    container_name: kafka1
    networks:
      - kafka
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
    restart: always

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    container_name: kafka-ui
    networks:
      - kafka
    environment:
      DYNAMIC_CONFIG_ENABLED: 'true'
      KAFKA_CLUSTERS_0_NAME: test1
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka1:9092
    ports:
      - 8080:8080
    restart: always

  connect-1:
    image: westlife0615/debezium-connector:1
    hostname: connect-1
    container_name: connect-1
    networks:
      - kafka
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka1:9092
      CONNECT_REST_ADVERTISED_HOST_NAME: connect-1
      CONNECT_GROUP_ID: connect-cluster
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
    restart: always

networks:
  kafka:
    driver: bridge
    external: true
EOF


docker-compose -f /tmp/docker-compose.yaml -p kafka-connect up -d