Prometheus

Dockerfile

@@ -18,7 +18,7 @@ WORKDIR /tmp
 
 # install required packages
 RUN apk add --no-cache \
-    gnupg 
+    gnupg
 
 # download and verify
 RUN set -eux; \
@@ -35,7 +35,6 @@ RUN set -eux; \
   tar -xzf ${FILENAME}; \
   mv kafka_${FULL_VER} kafka
 
-
 # Stage: Final Build 
 FROM openjdk:11-jre-slim
 

Dockerfile-prometheus-exporter

@@ -0,0 +1,64 @@
+# Define Build Arguments
+ARG INSTALL_DIR=/opt
+ARG SCALA_VER=2.13
+ARG KAFKA_VER=2.7.0
+
+# Stage: Download and Verify
+FROM alpine AS downloader
+
+ARG SCALA_VER
+ARG KAFKA_VER
+ARG FULL_VER=${SCALA_VER}-${KAFKA_VER}
+
+ARG FILENAME=kafka_${FULL_VER}.tgz
+ARG URL=https://downloads.apache.org/kafka/${KAFKA_VER}/${FILENAME}
+ARG KEYS_URL=https://downloads.apache.org/kafka/KEYS
+ARG JMX_PROM_EXPORTER=https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+
+WORKDIR /tmp
+
+# install required packages
+RUN apk add --no-cache \
+    gnupg
+
+# download and verify
+RUN set -eux; \
+  wget -q -O ${FILENAME} "${URL}"
+
+RUN set -eux; \
+  wget -q -O ${FILENAME}.asc ${URL}.asc; \
+  wget -q -O KEYS "${KEYS_URL}"; \
+  gpg --import KEYS; \
+  gpg --batch --verify ${FILENAME}.asc ${FILENAME}
+
+RUN set -eux; \
+  echo ${FULL_VER} > thebinary.kafka.version; \
+  tar -xzf ${FILENAME}; \
+  mv kafka_${FULL_VER} kafka
+
+# JMX prometheus exporter
+RUN mkdir monitoring
+RUN wget -q -O monitoring/jmx_prometheus_javaagent.jar "${JMX_PROM_EXPORTER}"
+COPY ./jmx_prometheus_exporter/config.yaml monitoring/config.yaml
+
+
+# Stage: Final Build 
+FROM openjdk:11-jre-slim
+
+ARG INSTALL_DIR
+
+COPY --from=downloader /tmp/thebinary.kafka.version /etc/thebinary.kafka.version
+COPY --from=downloader /tmp/kafka ${INSTALL_DIR}/kafka
+COPY --from=downloader /tmp/monitoring ${INSTALL_DIR}/kafka/monitoring
+
+ADD kafka-exec.sh /usr/bin/
+
+RUN set -eux; \
+  echo ${INSTALL_DIR}/kafka > /etc/thebinary.kafka.install.dir; \
+  cd /usr/bin/ && for sh in $(ls ${INSTALL_DIR}/kafka/bin/); do ln -s kafka-exec.sh $sh; done; \
+  mkdir -p /etc/kafka; \
+  cp -a ${INSTALL_DIR}/kafka/config/server.properties /etc/kafka/server.properties
+
+EXPOSE 9092/tcp
+
+CMD kafka-server-start.sh /etc/kafka/server.properties

README.md

@@ -31,6 +31,14 @@ zookeeper.connect=zookeeper:2181
 ```sh
 docker run --network kafka -it -d -v $(PWD)/server.properties:/etc/kafka/config/server.properties thebinary/kafka
 ```
+### Enable Prometheus Exporter for Kafka Metrics
+4. Enable Prometheus Exporter(Optional)
+Export the following environment variable while starting the container.
+This makes the container listen on port 8080 for jmx exporter.
+Change the port number in the KAFKA_OPTS env variable to use a different port than 8080.
+```
+KAFKA_OPTS=-javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+```
 
 ## Use as Kafka Tools
 

compose/multi-broker/docker-compose.yml

@@ -43,4 +43,4 @@ services:
     volumes:
       - type: bind
         source: ./broker2
-        target: /etc/kafka
+        target: /etc/kafka

compose/prometheus-exporter/multi-broker/broker0/server.properties

@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=3
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181

compose/prometheus-exporter/multi-broker/broker1/server.properties

@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=1
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=2
+transaction.state.log.replication.factor=2
+transaction.state.log.min.isr=2
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181

compose/prometheus-exporter/multi-broker/broker2/server.properties

@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=2
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=3
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181

compose/prometheus-exporter/multi-broker/docker-compose.yml

@@ -0,0 +1,62 @@
+version: "3.9"
+services:
+  zookeeper:
+    image: zookeeper
+
+  kafka0:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker0
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
+  kafka1:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker1
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
+  kafka2:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker2
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+

compose/prometheus-exporter/single-broker/config/server.properties

@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181

compose/prometheus-exporter/single-broker/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3.9"
+services:
+  zookeeper:
+    image: zookeeper
+
+  kafka:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./config
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml