docker-compose.yml

name: kafka-modern

services:
  kdc:
    image: plt-kafka-kdc:latest
    pull_policy: never
    build:
      context: ./docker/kerberos
    ports:
      - '${KAFKA_KDC_PORT:-8000}:88/tcp'
      - '${KAFKA_KDC_PORT:-8000}:88/udp'
      - '${KAFKA_KADMIN_PORT:-8001}:749'
    volumes:
      - './docker/kerberos/krb5-kdc.conf:/etc/krb5.conf:ro'
      - './docker/kerberos/kdc.conf:/var/lib/krb5kdc/kdc.conf:ro'
      - './docker/kerberos/init.sh:/init.sh:ro'
      - 'kerberos-data:/data'
    entrypoint: ['/bin/sh', '/init.sh']
    healthcheck:
      test: ['CMD', 'kadmin.local', '-q', 'list_principals']
      interval: 10s
      timeout: 5s
      retries: 5

  schema-registry:
    image: confluentinc/cp-schema-registry:${SCHEMA_REGISTRY_VERSION:-8.1.0}
    ports:
      - '${SCHEMA_REGISTRY_PORT:-8004}:8081'
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:8081/subjects']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker-single:19092
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC_REPLICATION_FACTOR: 1
      SCHEMA_REGISTRY_DEBUG: true
      SCHEMA_REGISTRY_SCHEMA_COMPATIBILITY_LEVEL: BACKWARD
      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_ORIGIN: '*'
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_METHODS: GET,POST,PUT,DELETE,OPTIONS

  schema-registry-auth:
    image: nginx:1.29.3-alpine@sha256:b3c656d55d7ad751196f21b7fd2e8d4da9cb430e32f646adcf92441b72f82b14
    depends_on:
      schema-registry:
        condition: service_healthy
    ports:
      - '${SCHEMA_REGISTRY_AUTH_PORT:-8005}:8081'
      - '${SCHEMA_REGISTRY_BEARER_PORT:-8006}:8082'
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:8080']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    volumes:
      - './docker/schema-registry-auth/nginx.conf:/etc/nginx/nginx.conf:ro'
      - './docker/schema-registry-auth/passwd:/etc/nginx/.htpasswd:ro'

  broker-single:
    # Rule of thumb: Confluent Kafka Version = Apache Kafka Version + 4.0.0
    image: &image confluentinc/cp-kafka:${KAFKA_VERSION:-8.1.0}
    ports:
      - '${KAFKA_SINGLE_PORT:-9001}:9092'
    healthcheck: &health_check # Health_check
      test: ['CMD', '/usr/bin/kafka-broker-api-versions', '--bootstrap-server', 'localhost:19092']
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s
    environment: &common_config
      CLUSTER_ID: 'gPJsB-TJE9oxCMwjUbji7Q'
      KAFKA_PROCESS_ROLES: 'broker,controller'
      # Ports configuration
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'DOCKER'
      # Broker specific general and port options
      KAFKA_NODE_ID: '1'
      KAFKA_LISTENERS: 'PLAINTEXT://:9092,DOCKER://:19092,CONTROLLER://:29092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'PLAINTEXT:PLAINTEXT,DOCKER:PLAINTEXT,CONTROLLER:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:${KAFKA_SINGLE_PORT:-9001},DOCKER://broker-single:19092'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-single:29092'
      # Replication options
      KAFKA_DEFAULT_REPLICATION_FACTOR: '1'
      KAFKA_MIN_INSYNC_REPLICAS: '1'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
      KAFKA_TRANSACTION_PARTITION_VERIFICATION_ENABLE: 'false'
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
      # Consumer group options
      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
      KAFKA_GROUP_CONSUMER_HEARTBEAT_INTERVAL_MS: '200'
      KAFKA_GROUP_CONSUMER_MIN_HEARTBEAT_INTERVAL_MS: '100'

  broker-sasl:
    image: *image
    ports:
      - '${KAFKA_SASL_PORT:-9002}:9092' # SASL
      - '${KAFKA_SASL_PLAIN_PORT:-19002}:19092' # PLAIN TEXT - Used to create users
    healthcheck: *health_check
    volumes:
      - './docker/sasl/jaas.conf:/etc/kafka/jaas.conf:ro'
    environment:
      <<: *common_config
      # Broker specific general and port options
      KAFKA_LISTENERS: 'SASL://:9092,DOCKER://:19092,CONTROLLER://:29092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SASL:SASL_PLAINTEXT,DOCKER:PLAINTEXT,CONTROLLER:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'SASL://localhost:${KAFKA_SASL_PORT:-9002},DOCKER://broker-sasl:19092'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-sasl:29092'
      # SASL
      KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/jaas.conf'
      KAFKA_LISTENER_NAME_SASL_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin" user_admin="admin";'
      KAFKA_LISTENER_NAME_SASL_SCRAM___SHA___256_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin";'
      KAFKA_LISTENER_NAME_SASL_SCRAM___SHA___512_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.scram.ScramLoginModule required username="admin" password="admin";'
      KAFKA_LISTENER_NAME_SASL_OAUTHBEARER_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;'
      KAFKA_CONNECTIONS_MAX_REAUTH_MS: 5000
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'false'
      KAFKA_SUPER_USERS: 'User:admin'
      KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,OAUTHBEARER,SCRAM-SHA-256,SCRAM-SHA-512'
      KAFKA_SASL_MECHANISM_CONTROLLER_PROTOCOL: 'PLAIN'
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
      KAFKA_SASL_OAUTHBEARER_EXPECTED_ISSUER: kafka
      KAFKA_SASL_OAUTHBEARER_EXPECTED_AUDIENCE: users
      KAFKA_SASL_OAUTHBEARER_EXPECTED_SCOPE: test

  broker-sasl-kerberos:
    image: *image
    ports:
      - '${KAFKA_KERBEROS_PORT:-9003}:9092' # SASL
      - '${KAFKA_KERBEROS_PLAIN_PORT:-19003}:19092' # PLAIN TEXT - Used to create users
    healthcheck: *health_check
    volumes:
      - './docker/sasl/jaas-kerberos.conf:/etc/kafka/jaas.conf:ro'
      - './docker/kerberos/krb5-broker.conf:/etc/krb5.conf:ro'
      - 'kerberos-data:/etc/kafka/keytabs:ro'
    depends_on:
      kdc:
        condition: service_healthy
    environment:
      <<: *common_config
      # Broker specific general and port options
      KAFKA_LISTENERS: 'SASL://:9092,DOCKER://:19092,CONTROLLER://:29092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SASL:SASL_PLAINTEXT,DOCKER:PLAINTEXT,CONTROLLER:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'SASL://localhost:${KAFKA_KERBEROS_PORT:-9003},DOCKER://broker-sasl-kerberos:19092'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-sasl-kerberos:29092'
      # SASL
      KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/jaas.conf -Djava.security.krb5.conf=/etc/krb5.conf'
      KAFKA_LISTENER_NAME_SASL_GSSAPI_SASL_JAAS_CONFIG: 'com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab="/etc/kafka/keytabs/broker.keytab" principal="broker/broker-sasl-kerberos@EXAMPLE.COM";'
      KAFKA_CONNECTIONS_MAX_REAUTH_MS: 5000
      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'false'
      KAFKA_SUPER_USERS: 'User:admin;User:broker/broker-sasl-kerberos@EXAMPLE.COM;User:admin-keytab/localhost@EXAMPLE.COM;User:admin-password/localhost@EXAMPLE.COM'
      KAFKA_SASL_ENABLED_MECHANISMS: 'GSSAPI'
      KAFKA_SASL_MECHANISM_CONTROLLER_PROTOCOL: 'PLAIN'
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
      KAFKA_SASL_KERBEROS_SERVICE_NAME: 'kafka'

  broker-cluster-1:
    image: *image
    ports:
      - '${KAFKA_CLUSTER_1_PORT:-9011}:9092'
    healthcheck: *health_check
    environment: &cluster_config
      <<: *common_config
      # Broker specific general and port options
      KAFKA_NODE_ID: '1'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:${KAFKA_CLUSTER_1_PORT:-9011},DOCKER://broker-cluster-1:19092'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker-cluster-1:29092,2@broker-cluster-2:29092,3@broker-cluster-3:29092'
      # Replication options
      KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '3'
      KAFKA_CFG_MIN_INSYNC_REPLICAS: '1'
      KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: '3'
      KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR: '1'
      KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '3'
      # Consumer group options
      KAFKA_CFG_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer'
      KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
      # ACL options
      KAFKA_AUTHORIZER_CLASS_NAME: 'org.apache.kafka.metadata.authorizer.StandardAuthorizer'
      KAFKA_SUPER_USERS: 'User:ANONYMOUS'

  broker-cluster-2:
    image: *image
    ports:
      - '${KAFKA_CLUSTER_2_PORT:-9012}:9092'
    healthcheck: *health_check
    environment:
      <<: *cluster_config
      # Broker specific general and port options
      KAFKA_NODE_ID: '2'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:${KAFKA_CLUSTER_2_PORT:-9012},DOCKER://broker-cluster-2:19092'

  broker-cluster-3:
    image: *image
    ports:
      - '${KAFKA_CLUSTER_3_PORT:-9013}:9092'
    healthcheck: *health_check
    environment:
      <<: *cluster_config
      # Broker specific general and port options
      KAFKA_NODE_ID: '3'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:${KAFKA_CLUSTER_3_PORT:-9013},DOCKER://broker-cluster-3:19092'

volumes:
  kerberos-data:
    driver: local
    driver_opts:
      type: tmpfs
      device: tmpfs
      o: size=16m,mode=0755