From d70df90ae28b63280db80b0a372b252967156c75 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Mon, 1 Nov 2021 17:15:25 +0800 Subject: [PATCH] Integrate prometheus service --- README.md | 32 ++++++++++++++--- docker/start_broker.sh | 40 +++++++++++++++------- docker/start_kafka_tool.sh | 14 ++++---- docker/start_prometheus.sh | 70 ++++++++++++++++++++++++++++++++++++++ docker/start_zookeeper.sh | 12 +++---- 5 files changed, 140 insertions(+), 28 deletions(-) create mode 100755 docker/start_prometheus.sh diff --git a/README.md b/README.md index f45af32a61..362417a0a9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ java -jar astraea-0.0.1-alpha.1-all.jar [tool] [args] The following scripts can build a kafka cluster by containers in one minute. -### Set up zookeeper +### Run Zookeeper ```shell ./docker/start_zookeeper.sh @@ -46,7 +46,7 @@ run ./docker/start_broker.sh zookeeper.connect=192.168.50.178:17228 to join kafk You can define `ZOOKEEPER_VERSION` to change the binary version. -### Set up (kafka) broker +### Run Kafka Broker After the zk env is running, you can copy the command (see above example) from zk script output to set up kafka. For example: ```shell @@ -57,14 +57,38 @@ The console will show the broker connection information and JMX address. For exa ```shell ================================================= -broker address: 192.168.50.224:11248 -jmx address: 192.168.50.224:15905 +broker address: 192.168.50.178:12747 +jmx address: 192.168.50.178:10216 +exporter address: 192.168.50.178:10558 +broker id: 677 ================================================= ``` +1. `broker address` is used by kafka client code. The alias is bootstrap server. +2. `jmx address` exports the java metrics by JMX +3. `exporter address` is the address of prometheus exporter. + The command to set up broker can be executed multiple times to create a broker cluster. The env `KAFKA_VERSION` is used to define the release version of kafka. Or you can define `KAFKA_REVISION` to run kafka based on specific revision of source code. +### Run Prometheus + +The exporters of brokers can offer metrics to Prometheus to monitor cluster status. For example, your kafka cluster has +two brokers. The exporter address of first broker is `192.168.50.178:10558` and another is `192.168.50.178:10553`. You can +use following command to run a prometheus service. + +```shell +./docker/start_prometheus.sh 192.168.50.178:10558,192.168.50.178:10553 +``` + +The console will show the http address of prometheus service. +```shell +================================================= +config file: /tmp/prometheus-15483.yml +prometheus address: http://192.168.50.178:15483 +================================================= +``` + --- ## Performance Benchmark diff --git a/docker/start_broker.sh b/docker/start_broker.sh index f33c3cf813..d18574ef24 100755 --- a/docker/start_broker.sh +++ b/docker/start_broker.sh @@ -23,7 +23,6 @@ function showHelp() { echo " num.network.threads=10 set JVM memory" echo " memory=\"-Xmx2G -Xms2G\" set JVM memory" } - # ===================================================================== if [[ "$(which docker)" == "" ]]; then @@ -35,7 +34,9 @@ if [[ -z "$KAFKA_VERSION" ]]; then KAFKA_VERSION=2.8.1 fi -USER=astraea +kafka_user=astraea +exporter_version="0.16.1" +exporter_port="$(($(($RANDOM % 10000)) + 10000))" image_name=astraea/broker:$KAFKA_VERSION if [[ -n "$KAFKA_REVISION" ]]; then image_name=astraea/broker:$KAFKA_REVISION @@ -133,19 +134,25 @@ FROM ubuntu:20.04 RUN apt-get update && apt-get upgrade -y && DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-11-jdk wget git curl # add user -RUN groupadd $USER && useradd -ms /bin/bash -g $USER $USER +RUN groupadd $kafka_user && useradd -ms /bin/bash -g $kafka_user $kafka_user # change user -USER $USER +USER $kafka_user + +# download jmx exporter +RUN mkdir /tmp/jmx_exporter +WORKDIR /tmp/jmx_exporter +RUN wget https://raw.githubusercontent.com/prometheus/jmx_exporter/master/example_configs/kafka-2_0_0.yml +RUN wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${exporter_version}/jmx_prometheus_javaagent-${exporter_version}.jar # build kafka from source code RUN git clone https://github.com/apache/kafka /tmp/kafka WORKDIR /tmp/kafka RUN git checkout $KAFKA_REVISION RUN ./gradlew clean releaseTarGz -RUN mkdir /home/$USER/kafka -RUN tar -zxvf \$(find ./core/build/distributions/ -maxdepth 1 -type f -name kafka_*SNAPSHOT.tgz) -C /home/$USER/kafka --strip-components=1 -WORKDIR "/home/$USER/kafka" +RUN mkdir /home/$kafka_user/kafka +RUN tar -zxvf \$(find ./core/build/distributions/ -maxdepth 1 -type f -name kafka_*SNAPSHOT.tgz) -C /home/$kafka_user/kafka --strip-components=1 +WORKDIR "/home/$kafka_user/kafka" Dockerfile @@ -157,17 +164,23 @@ FROM ubuntu:20.04 RUN apt-get update && apt-get upgrade -y && apt-get install -y openjdk-11-jdk wget # add user -RUN groupadd $USER && useradd -ms /bin/bash -g $USER $USER +RUN groupadd $kafka_user && useradd -ms /bin/bash -g $kafka_user $kafka_user # change user -USER $USER +USER $kafka_user + +# download jmx exporter +RUN mkdir /tmp/jmx_exporter +WORKDIR /tmp/jmx_exporter +RUN wget https://raw.githubusercontent.com/prometheus/jmx_exporter/master/example_configs/kafka-2_0_0.yml +RUN wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/${exporter_version}/jmx_prometheus_javaagent-${exporter_version}.jar # download kafka WORKDIR /tmp RUN wget https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz -RUN mkdir /home/$USER/kafka -RUN tar -zxvf kafka_2.13-${KAFKA_VERSION}.tgz -C /home/$USER/kafka --strip-components=1 -WORKDIR "/home/$USER/kafka" +RUN mkdir /home/$kafka_user/kafka +RUN tar -zxvf kafka_2.13-${KAFKA_VERSION}.tgz -C /home/$kafka_user/kafka --strip-components=1 +WORKDIR "/home/$kafka_user/kafka" Dockerfile fi @@ -175,13 +188,16 @@ fi docker run -d \ -e KAFKA_HEAP_OPTS="$KAFKA_HEAP" \ -e KAFKA_JMX_OPTS="$jmx_opts" \ + -e KAFKA_OPTS="-javaagent:/tmp/jmx_exporter/jmx_prometheus_javaagent-${exporter_version}.jar=$exporter_port:/tmp/jmx_exporter/kafka-2_0_0.yml" \ -v $config_file:/tmp/broker.properties:ro \ -p $broker_port:9092 \ -p $broker_jmx_port:$broker_jmx_port \ + -p $exporter_port:$exporter_port \ $image_name ./bin/kafka-server-start.sh /tmp/broker.properties echo "=================================================" echo "broker address: ${address}:$broker_port" echo "jmx address: ${address}:$broker_jmx_port" +echo "exporter address: ${address}:$exporter_port" echo "broker id: $broker_id" echo "=================================================" diff --git a/docker/start_kafka_tool.sh b/docker/start_kafka_tool.sh index 0baf57addc..7b68c854ca 100755 --- a/docker/start_kafka_tool.sh +++ b/docker/start_kafka_tool.sh @@ -1,8 +1,10 @@ #!/bin/bash +# =============================[functions]============================= function showHelp() { echo "Usage: [ KAFKA SCRIPT ] [ OPTIONS ]" } +# ===================================================================== if [[ "$(which docker)" == "" ]]; then echo "you have to install docker" @@ -26,7 +28,7 @@ image_name=astraea/kafka-tool:$KAFKA_VERSION # set JVM heap KAFKA_HEAP="${KAFKA_HEAP:-"-Xmx2G -Xms2G"}" -USER=astraea +kafka_user=astraea docker build -t $image_name - < "$file" +global: + scrape_interval: 15s + external_labels: + monitor: 'kafka-monitor' + +scrape_configs: + - job_name: 'brokers' + scrape_interval: 5s + static_configs: + - targets: [$targets] + +EOT + +docker run -d \ + -p $prometheus_port:9090 \ + -v $file:/etc/prometheus/prometheus.yml \ + prom/prometheus + +echo "=================================================" +echo "config file: $file" +echo "prometheus address: http://${address}:$prometheus_port" +echo "=================================================" \ No newline at end of file diff --git a/docker/start_zookeeper.sh b/docker/start_zookeeper.sh index 7c3ed78993..191ff45703 100755 --- a/docker/start_zookeeper.sh +++ b/docker/start_zookeeper.sh @@ -24,7 +24,7 @@ if [[ -z "$ZOOKEEPER_VERSION" ]]; then ZOOKEEPER_VERSION=3.6.3 fi -USER=astraea +zookeeper_user=astraea image_name=astraea/zookeeper:$ZOOKEEPER_VERSION zk_port="$(($(($RANDOM % 10000)) + 10000))" address=$(getAddress) @@ -36,17 +36,17 @@ FROM ubuntu:20.04 RUN apt-get update && apt-get upgrade -y && apt-get install -y openjdk-11-jdk wget # add user -RUN groupadd $USER && useradd -ms /bin/bash -g $USER $USER +RUN groupadd $zookeeper_user && useradd -ms /bin/bash -g $zookeeper_user $zookeeper_user # change user -USER $USER +USER $zookeeper_user # download zookeeper WORKDIR /tmp RUN wget https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -RUN mkdir /home/$USER/zookeeper -RUN tar -zxvf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /home/$USER/zookeeper --strip-components=1 -WORKDIR /home/$USER/zookeeper +RUN mkdir /home/$zookeeper_user/zookeeper +RUN tar -zxvf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /home/$zookeeper_user/zookeeper --strip-components=1 +WORKDIR /home/$zookeeper_user/zookeeper # create config file RUN echo "tickTime=2000" >> ./conf/zoo.cfg