From ca0ec09e2dcd53b225255f6fd05a3e58e0e6f3ab Mon Sep 17 00:00:00 2001
From: rhythmic <rhythmic@trouble.rhythmicsoul>
Date: Wed, 3 Mar 2021 16:14:41 +0545
Subject: [PATCH 1/5] added the support for prometheus jmx_exporter

---
 Dockerfile                               |  9 ++-
 compose/multi-broker/docker-compose.yml  | 10 +++
 compose/single-broker/docker-compose.yml |  3 +
 jmx_prometheus_exporter/config.yaml      | 88 ++++++++++++++++++++++++
 4 files changed, 109 insertions(+), 1 deletion(-)
 create mode 100644 jmx_prometheus_exporter/config.yaml

diff --git a/Dockerfile b/Dockerfile
index 8e852e6..d311a10 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,12 +13,13 @@ ARG FULL_VER=${SCALA_VER}-${KAFKA_VER}
 ARG FILENAME=kafka_${FULL_VER}.tgz
 ARG URL=https://downloads.apache.org/kafka/${KAFKA_VER}/${FILENAME}
 ARG KEYS_URL=https://downloads.apache.org/kafka/KEYS
+ARG JMX_PROM_EXPORTER=https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
 
 WORKDIR /tmp
 
 # install required packages
 RUN apk add --no-cache \
-    gnupg 
+    gnupg
 
 # download and verify
 RUN set -eux; \
@@ -35,6 +36,11 @@ RUN set -eux; \
   tar -xzf ${FILENAME}; \
   mv kafka_${FULL_VER} kafka
 
+# JMX prometheus exporter
+RUN mkdir monitoring
+RUN wget -q -O monitoring/jmx_prometheus_javaagent.jar "${JMX_PROM_EXPORTER}"
+COPY ./jmx_prometheus_exporter/config.yaml monitoring/config.yaml
+
 
 # Stage: Final Build 
 FROM openjdk:11-jre-slim
@@ -43,6 +49,7 @@ ARG INSTALL_DIR
 
 COPY --from=downloader /tmp/thebinary.kafka.version /etc/thebinary.kafka.version
 COPY --from=downloader /tmp/kafka ${INSTALL_DIR}/kafka
+COPY --from=downloader /tmp/monitoring ${INSTALL_DIR}/kafka/monitoring
 
 ADD kafka-exec.sh /usr/bin/
 
diff --git a/compose/multi-broker/docker-compose.yml b/compose/multi-broker/docker-compose.yml
index 36d30ee..175ed21 100644
--- a/compose/multi-broker/docker-compose.yml
+++ b/compose/multi-broker/docker-compose.yml
@@ -16,6 +16,9 @@ services:
       - type: bind
         source: ./broker0
         target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
 
   kafka1:
     depends_on:
@@ -30,6 +33,9 @@ services:
       - type: bind
         source: ./broker1
         target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
 
   kafka2:
     depends_on:
@@ -44,3 +50,7 @@ services:
       - type: bind
         source: ./broker2
         target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
diff --git a/compose/single-broker/docker-compose.yml b/compose/single-broker/docker-compose.yml
index 54ecaba..a7d1839 100644
--- a/compose/single-broker/docker-compose.yml
+++ b/compose/single-broker/docker-compose.yml
@@ -16,3 +16,6 @@ services:
       - type: bind
         source: ./config
         target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
diff --git a/jmx_prometheus_exporter/config.yaml b/jmx_prometheus_exporter/config.yaml
new file mode 100644
index 0000000..5f35f37
--- /dev/null
+++ b/jmx_prometheus_exporter/config.yaml
@@ -0,0 +1,88 @@
+lowercaseOutputName: true
+rules:
+- pattern : kafka.cluster<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+  name: kafka_cluster_$1_$2
+  labels:
+    topic: "$3"
+    partition: "$4"
+- pattern : kafka.log<type=Log, name=(.+), topic=(.+), partition=(.+)><>Value
+  name: kafka_log_$1
+  labels:
+    topic: "$2"
+    partition: "$3"
+- pattern : kafka.controller<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_controller_$1_$2
+- pattern : kafka.network<type=(.+), name=(.+)><>Value
+  name: kafka_network_$1_$2
+- pattern : kafka.network<type=(.+), name=(.+)PerSec, request=(.+)><>Count
+  name: kafka_network_$1_$2_total
+  labels:
+    request: "$3"
+- pattern : kafka.network<type=(.+), name=(\w+), networkProcessor=(.+)><>Count
+  name: kafka_network_$1_$2
+  labels:
+    request: "$3"
+  type: COUNTER
+- pattern : kafka.network<type=(.+), name=(\w+), request=(\w+)><>Count
+  name: kafka_network_$1_$2
+  labels:
+    request: "$3"
+- pattern : kafka.network<type=(.+), name=(\w+)><>Count
+  name: kafka_network_$1_$2
+- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
+  name: kafka_server_$1_$2_total
+  labels:
+    topic: "$3"
+- pattern : kafka.server<type=(.+), name=(.+)PerSec\w*><>Count
+  name: kafka_server_$1_$2_total
+  type: COUNTER
+
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+    topic: "$4"
+    partition: "$5"
+- pattern : kafka.server<type=(.+), name=(.+), topic=(.+), partition=(.*)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    topic: "$3"
+    partition: "$4"
+- pattern : kafka.server<type=(.+), name=(.+), topic=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    topic: "$3"
+  type: COUNTER
+
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+    broker: "$4:$5"
+- pattern : kafka.server<type=(.+), name=(.+), clientId=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+  labels:
+    clientId: "$3"
+- pattern : kafka.server<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_server_$1_$2
+
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*><>Count
+  name: kafka_$1_$2_$3_total
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  labels:
+    topic: "$4"
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+)PerSec\w*, topic=(.+), partition=(.+)><>Count
+  name: kafka_$1_$2_$3_total
+  labels:
+    topic: "$4"
+    partition: "$5"
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+)><>(Count|Value)
+  name: kafka_$1_$2_$3_$4
+  type: COUNTER
+- pattern : kafka.(\w+)<type=(.+), name=(.+), (\w+)=(.+)><>(Count|Value)
+  name: kafka_$1_$2_$3_$6
+  labels:
+    "$4": "$5"

From f733a5669710a755e55a1ddfd87be30b8a19b3a5 Mon Sep 17 00:00:00 2001
From: rhythmic <rhythmic@trouble.rhythmicsoul>
Date: Wed, 3 Mar 2021 16:24:20 +0545
Subject: [PATCH 2/5] added the Section for enabling jmx exporter

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index 5bbb40c..0146858 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,13 @@ zookeeper.connect=zookeeper:2181
 ```sh
 docker run --network kafka -it -d -v $(PWD)/server.properties:/etc/kafka/config/server.properties thebinary/kafka
 ```
+### Enable Prometheus Exporter for Kafka Metrics
+4. Enable Prometheus Exporter(Optional)
+Export the following environment variable while starting the container.
+This makes the container listen on port 8080 for jmx exporter.
+```
+KAFKA_OPTS=-javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+```
 
 ## Use as Kafka Tools
 

From 475b05ce44b664a7992e5068aa680b8ed8136d8b Mon Sep 17 00:00:00 2001
From: rhythmic <rhythmic@trouble.rhythmicsoul>
Date: Wed, 3 Mar 2021 16:28:19 +0545
Subject: [PATCH 3/5] changed the readme file for changing the exporter port

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 0146858..df8b107 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,7 @@ docker run --network kafka -it -d -v $(PWD)/server.properties:/etc/kafka/config/
 4. Enable Prometheus Exporter(Optional)
 Export the following environment variable while starting the container.
 This makes the container listen on port 8080 for jmx exporter.
+Change the port number in the KAFKA_OPTS env variable to use a different port than 8080.
 ```
 KAFKA_OPTS=-javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
 ```

From f93126eb49bf1f4429e31b12ca37d2e1e2fe7ccb Mon Sep 17 00:00:00 2001
From: rhythmic <rhythmic@trouble.rhythmicsoul>
Date: Thu, 6 May 2021 10:24:07 +0545
Subject: [PATCH 4/5] added the separate docker compose files and configs along
 with docker file for prometheus exporter

---
 Dockerfile                                    |  7 --
 Dockerfile-prometheus-exporter                | 64 +++++++++++++++++++
 compose/multi-broker/docker-compose.yml       | 12 +---
 .../multi-broker/broker0/server.properties    | 31 +++++++++
 .../multi-broker/broker1/server.properties    | 31 +++++++++
 .../multi-broker/broker2/server.properties    | 31 +++++++++
 .../multi-broker/docker-compose.yml           | 62 ++++++++++++++++++
 .../single-broker/config/server.properties    | 31 +++++++++
 .../single-broker/docker-compose.yml          | 23 +++++++
 compose/single-broker/docker-compose.yml      |  3 -
 10 files changed, 274 insertions(+), 21 deletions(-)
 create mode 100644 Dockerfile-prometheus-exporter
 create mode 100644 compose/prometheus-exporter/multi-broker/broker0/server.properties
 create mode 100644 compose/prometheus-exporter/multi-broker/broker1/server.properties
 create mode 100644 compose/prometheus-exporter/multi-broker/broker2/server.properties
 create mode 100644 compose/prometheus-exporter/multi-broker/docker-compose.yml
 create mode 100644 compose/prometheus-exporter/single-broker/config/server.properties
 create mode 100644 compose/prometheus-exporter/single-broker/docker-compose.yml

diff --git a/Dockerfile b/Dockerfile
index d311a10..b1b49e8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,6 @@ ARG FULL_VER=${SCALA_VER}-${KAFKA_VER}
 ARG FILENAME=kafka_${FULL_VER}.tgz
 ARG URL=https://downloads.apache.org/kafka/${KAFKA_VER}/${FILENAME}
 ARG KEYS_URL=https://downloads.apache.org/kafka/KEYS
-ARG JMX_PROM_EXPORTER=https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
 
 WORKDIR /tmp
 
@@ -36,12 +35,6 @@ RUN set -eux; \
   tar -xzf ${FILENAME}; \
   mv kafka_${FULL_VER} kafka
 
-# JMX prometheus exporter
-RUN mkdir monitoring
-RUN wget -q -O monitoring/jmx_prometheus_javaagent.jar "${JMX_PROM_EXPORTER}"
-COPY ./jmx_prometheus_exporter/config.yaml monitoring/config.yaml
-
-
 # Stage: Final Build 
 FROM openjdk:11-jre-slim
 
diff --git a/Dockerfile-prometheus-exporter b/Dockerfile-prometheus-exporter
new file mode 100644
index 0000000..d311a10
--- /dev/null
+++ b/Dockerfile-prometheus-exporter
@@ -0,0 +1,64 @@
+# Define Build Arguments
+ARG INSTALL_DIR=/opt
+ARG SCALA_VER=2.13
+ARG KAFKA_VER=2.7.0
+
+# Stage: Download and Verify
+FROM alpine AS downloader
+
+ARG SCALA_VER
+ARG KAFKA_VER
+ARG FULL_VER=${SCALA_VER}-${KAFKA_VER}
+
+ARG FILENAME=kafka_${FULL_VER}.tgz
+ARG URL=https://downloads.apache.org/kafka/${KAFKA_VER}/${FILENAME}
+ARG KEYS_URL=https://downloads.apache.org/kafka/KEYS
+ARG JMX_PROM_EXPORTER=https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+
+WORKDIR /tmp
+
+# install required packages
+RUN apk add --no-cache \
+    gnupg
+
+# download and verify
+RUN set -eux; \
+  wget -q -O ${FILENAME} "${URL}"
+
+RUN set -eux; \
+  wget -q -O ${FILENAME}.asc ${URL}.asc; \
+  wget -q -O KEYS "${KEYS_URL}"; \
+  gpg --import KEYS; \
+  gpg --batch --verify ${FILENAME}.asc ${FILENAME}
+  
+RUN set -eux; \
+  echo ${FULL_VER} > thebinary.kafka.version; \
+  tar -xzf ${FILENAME}; \
+  mv kafka_${FULL_VER} kafka
+
+# JMX prometheus exporter
+RUN mkdir monitoring
+RUN wget -q -O monitoring/jmx_prometheus_javaagent.jar "${JMX_PROM_EXPORTER}"
+COPY ./jmx_prometheus_exporter/config.yaml monitoring/config.yaml
+
+
+# Stage: Final Build 
+FROM openjdk:11-jre-slim
+
+ARG INSTALL_DIR
+
+COPY --from=downloader /tmp/thebinary.kafka.version /etc/thebinary.kafka.version
+COPY --from=downloader /tmp/kafka ${INSTALL_DIR}/kafka
+COPY --from=downloader /tmp/monitoring ${INSTALL_DIR}/kafka/monitoring
+
+ADD kafka-exec.sh /usr/bin/
+
+RUN set -eux; \
+  echo ${INSTALL_DIR}/kafka > /etc/thebinary.kafka.install.dir; \
+  cd /usr/bin/ && for sh in $(ls ${INSTALL_DIR}/kafka/bin/); do ln -s kafka-exec.sh $sh; done; \
+  mkdir -p /etc/kafka; \
+  cp -a ${INSTALL_DIR}/kafka/config/server.properties /etc/kafka/server.properties
+
+EXPOSE 9092/tcp
+
+CMD kafka-server-start.sh /etc/kafka/server.properties
diff --git a/compose/multi-broker/docker-compose.yml b/compose/multi-broker/docker-compose.yml
index 175ed21..81332a2 100644
--- a/compose/multi-broker/docker-compose.yml
+++ b/compose/multi-broker/docker-compose.yml
@@ -16,9 +16,6 @@ services:
       - type: bind
         source: ./broker0
         target: /etc/kafka
-    environment:
-      # JMX Java agent for exporting kafka metrics to Prometheus
-      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
 
   kafka1:
     depends_on:
@@ -33,9 +30,6 @@ services:
       - type: bind
         source: ./broker1
         target: /etc/kafka
-    environment:
-      # JMX Java agent for exporting kafka metrics to Prometheus
-      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
 
   kafka2:
     depends_on:
@@ -49,8 +43,4 @@ services:
     volumes:
       - type: bind
         source: ./broker2
-        target: /etc/kafka
-    environment:
-      # JMX Java agent for exporting kafka metrics to Prometheus
-      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
-
+        target: /etc/kafka
\ No newline at end of file
diff --git a/compose/prometheus-exporter/multi-broker/broker0/server.properties b/compose/prometheus-exporter/multi-broker/broker0/server.properties
new file mode 100644
index 0000000..382e59d
--- /dev/null
+++ b/compose/prometheus-exporter/multi-broker/broker0/server.properties
@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=3
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181
diff --git a/compose/prometheus-exporter/multi-broker/broker1/server.properties b/compose/prometheus-exporter/multi-broker/broker1/server.properties
new file mode 100644
index 0000000..015e81d
--- /dev/null
+++ b/compose/prometheus-exporter/multi-broker/broker1/server.properties
@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=1
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=2
+transaction.state.log.replication.factor=2
+transaction.state.log.min.isr=2
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181
diff --git a/compose/prometheus-exporter/multi-broker/broker2/server.properties b/compose/prometheus-exporter/multi-broker/broker2/server.properties
new file mode 100644
index 0000000..218e436
--- /dev/null
+++ b/compose/prometheus-exporter/multi-broker/broker2/server.properties
@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=2
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=2
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=3
+transaction.state.log.replication.factor=3
+transaction.state.log.min.isr=3
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181
diff --git a/compose/prometheus-exporter/multi-broker/docker-compose.yml b/compose/prometheus-exporter/multi-broker/docker-compose.yml
new file mode 100644
index 0000000..0b3e511
--- /dev/null
+++ b/compose/prometheus-exporter/multi-broker/docker-compose.yml
@@ -0,0 +1,62 @@
+version: "3.9"
+services:
+  zookeeper:
+    image: zookeeper
+
+  kafka0:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker0
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
+  kafka1:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker1
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
+  kafka2:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./broker2
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
+
diff --git a/compose/prometheus-exporter/single-broker/config/server.properties b/compose/prometheus-exporter/single-broker/config/server.properties
new file mode 100644
index 0000000..a08f089
--- /dev/null
+++ b/compose/prometheus-exporter/single-broker/config/server.properties
@@ -0,0 +1,31 @@
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://:9092
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kafka-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=zookeeper:2181
diff --git a/compose/prometheus-exporter/single-broker/docker-compose.yml b/compose/prometheus-exporter/single-broker/docker-compose.yml
new file mode 100644
index 0000000..a41e902
--- /dev/null
+++ b/compose/prometheus-exporter/single-broker/docker-compose.yml
@@ -0,0 +1,23 @@
+version: "3.9"
+services:
+  zookeeper:
+    image: zookeeper
+
+  kafka:
+    depends_on:
+      - zookeeper
+    image: thebinary/kafka:2.13-2.7.0
+    build:
+      context: ../../../
+      dockerfile: Dockerfile-prometheus-exporter
+      args:
+        KAFKA_VER: 2.7.0
+        SCALA_VER: 2.13
+        JMX_PROM_EXPORTER: https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar
+    volumes:
+      - type: bind
+        source: ./config
+        target: /etc/kafka
+    environment:
+      # JMX Java agent for exporting kafka metrics to Prometheus
+      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml
diff --git a/compose/single-broker/docker-compose.yml b/compose/single-broker/docker-compose.yml
index a7d1839..54ecaba 100644
--- a/compose/single-broker/docker-compose.yml
+++ b/compose/single-broker/docker-compose.yml
@@ -16,6 +16,3 @@ services:
       - type: bind
         source: ./config
         target: /etc/kafka
-    environment:
-      # JMX Java agent for exporting kafka metrics to Prometheus
-      KAFKA_OPTS: -javaagent:/opt/kafka/monitoring/jmx_prometheus_javaagent.jar=8080:/opt/kafka/monitoring/config.yaml

From 48f3cbba4b4c4cc159fe02f7f9ae039e303d2ecf Mon Sep 17 00:00:00 2001
From: rhythmic <rhythmic@trouble.rhythmicsoul>
Date: Thu, 6 May 2021 10:35:34 +0545
Subject: [PATCH 5/5] removed the monitoring config in the docker build for
 normal image

---
 Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index b1b49e8..5119872 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,7 +42,6 @@ ARG INSTALL_DIR
 
 COPY --from=downloader /tmp/thebinary.kafka.version /etc/thebinary.kafka.version
 COPY --from=downloader /tmp/kafka ${INSTALL_DIR}/kafka
-COPY --from=downloader /tmp/monitoring ${INSTALL_DIR}/kafka/monitoring
 
 ADD kafka-exec.sh /usr/bin/