Debezium not able to create kafka topics - [mongodb, kafka, debezium] - Stack Overflow

admin管理员组

文章数量:1122847

I have been trying to setup Debezium with Mongodb but it's not working. Debezium is not able to create kafka topic. Not sure what the issue is. Tried using chatGpt also but to no avail. Need help with understanding whats missing in below mentioned docker file.

docker-compose.yml

services:

  #DEBEZIUM CONFIGURATION
  debezium:
    image: debezium/connect:1.4
    container_name: debezium
    environment:
      BOOTSTRAP_SERVERS: kafka:9092  # Use 'kafka' instead of 'localhost'
      GROUP_ID: 1
      CONFIG_STORAGE_TOPIC: connect_configs
      OFFSET_STORAGE_TOPIC: connect_offsets
#      KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
#      VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      DEBEZIUM_MONGODB_URI: "mongodb://mongo1:27017,mongo2:27018/?replicaSet=rs0"
#      DEBEZIUM_MONGODB_DB: "carbon_emission_db"
      CONNECT_TOPIC_CREATION_ENABLE: true
    depends_on:
      - kafka
      - mongo-1
    ports:
      - 8085:8085


  #KAFKA SERVICES CONFIGURATION
  schema-registry:
    container_name: schema-registry
    image: confluentinc/cp-schema-registry:7.4.4
    environment:
      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
      - SCHEMA_REGISTRY_HOST_NAME=schema-registry
      - SCHEMA_REGISTRY_LISTENERS=http://schema-registry:8081,http://localhost:8081
      #Enteries madeo fix scehema-registry connnection with kafka
      - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=kafka:9092  # Point to Kafka
      - SCHEMA_REGISTRY_LISTENER_HTTP_PORT=8081
      - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
    ports:
      - 8081:8081
    depends_on:
      - zookeeper
      - kafka

  zookeeper:
    image: quay.io/debezium/zookeeper
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 2181:2181

  kafka:
    image: confluentinc/cp-kafka:7.4.4
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - 9092:29092
    environment:
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

  kafka_manager:
    container_name: kafka-mngr
    image: kafkamanager/kafka-manager
    restart: always
    ports:
      - "9010:9010"
    depends_on:
      - zookeeper
      - kafka
    environment:
      ZK_HOSTS: "zookeeper:2181"
      APPLICATION_SECRET: "random-secret"

  #MONGODB CONFIGURATION
  #MONGODB-1
  mongo-1:
    image: mongo:7.0.5
    container_name: mongo1
    ports:
      - "27017:27017"
    command: mongod --replSet rs0 --port 27017 --bind_ip_all
#    command: ["--replSet", "rs0"]
    depends_on:
      - mongo-2
    volumes:
      - ./mongo-data:/data/db

  #MONGODB-2
  mongo-2:
    image: mongo:7.0.5
    container_name: mongo2
    ports:
      - "27018:27018"
    command: mongod --replSet rs0 --port 27018 --bind_ip_all

  #GRAFANA CONFIGURATION
  grafana:
    image: grafana/grafana:latest  # Specify a compatible version
    container_name: grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin  # Set the admin password
      - GF_INSTALL_PLUGINS=grafana-piechart-panel,grafana-clock-panel  # Optional plugins
    ports:
      - "3010:3000"  # Expose Grafana on port 3000
    volumes:
      - grafana-storage:/var/lib/grafana  # Persist data
    #depends_on:
    #  - mongo-1  # If you want to visualize MongoDB data, adjust dependencies accordingly

# Volume section for grafana
volumes:
  grafana-storage:

POC at following location:

本文标签： Debezium not able to create kafka topicsmongodbKafkadebeziumStack Overflow