Move config file format to YAML

.gitignore

@@ -8,9 +8,6 @@ logs/
 .settings/
 src/test/java/com/linkedin/xinfra/monitor/RandomTests.java
 
-config/andrew-choi.properties
-config/andrew-multi-cluster-monitor.properties
-
 kafka-monitor.iml
 kafka-monitor.ipr
 kafka-monitor.iws

README.md

@@ -99,8 +99,8 @@ Xinfra Monitor supports Apache Kafka 0.8 to 2.0:
 
 <ol>
 <li> We advise advanced users to run Xinfra Monitor with
-<code>./bin/xinfra-monitor-start.sh config/xinfra-monitor.properties</code>. The default
-xinfra-monitor.properties in the repo provides an simple example of how to
+<code>./bin/xinfra-monitor-start.sh config/xinfra-monitor.yaml</code>. The default
+xinfra-monitor.yaml in the repo provides an simple example of how to
 monitor a single cluster. You probably need to change the value of
 <code>zookeeper.connect</code> and <code>bootstrap.servers</code> to point to your cluster.
   </li>
@@ -109,7 +109,7 @@ monitor a single cluster. You probably need to change the value of
 Config class for respective service, e.g. ProduceServiceConfig.java and
 ConsumeServiceConfig.java.</li>
 <br />
-<li> You can specify multiple SingleClusterMonitor in the xinfra-monitor.properties to
+<li> You can specify multiple SingleClusterMonitor in the xinfra-monitor.yaml to
 monitor multiple Kafka clusters in one Xinfra Monitor process. As another
 advanced use-case, you can point ProduceService and ConsumeService to two different Kafka clusters that are connected by MirrorMaker to monitor their end-to-end latency.</li>
 <br />  
@@ -146,16 +146,16 @@ $ ./gradlew jar
 
 ### Start XinfraMonitor to run tests/services specified in the config file
 ```
-$ ./bin/xinfra-monitor-start.sh config/xinfra-monitor.properties
+$ ./bin/xinfra-monitor-start.sh config/xinfra-monitor.yaml
 ```
 
 ### Run Xinfra Monitor with arbitrary producer/consumer configuration (e.g. SASL enabled client)
-Edit `config/xinfra-monitor.properties` to specify custom configurations for producer in the key/value map `produce.producer.props` in
-`config/xinfra-monitor.properties`. Similarly specify configurations for
+Edit `config/xinfra-monitor.yaml` to specify custom configurations for producer in the key/value map `produce.producer.props` in
+`config/xinfra-monitor.yaml`. Similarly specify configurations for
 consumer as well. The documentation for producer and consumer in the key/value maps can be found in the Apache Kafka wiki.
 
 ```
-$ ./bin/xinfra-monitor-start.sh config/xinfra-monitor.properties
+$ ./bin/xinfra-monitor-start.sh config/xinfra-monitor.yaml
 ```
 
 ### Run SingleClusterMonitor app to monitor kafka cluster
@@ -169,16 +169,16 @@ $ ./bin/single-cluster-monitor.sh --topic test --broker-list localhost:9092 --zo
 ```
 
 ### Run MultiClusterMonitor app to monitor a pipeline of Kafka clusters connected by MirrorMaker
-Edit `config/multi-cluster-monitor.properties` to specify the right broker and
+Edit `config/multi-cluster-monitor.yaml` to specify the right broker and
 zookeeper url as suggested by the comment in the properties file
 
 Metrics `produce-availability-avg` and `consume-availability-avg` demonstrate
 whether messages can be properly produced to the source cluster and consumed
-from the destination cluster. See config/multi-cluster-monitor.properties for
+from the destination cluster. See config/multi-cluster-monitor.yaml for
 the full jmx path for these metrics.
 
 ```
-$ ./bin/xinfra-monitor-start.sh config/multi-cluster-monitor.properties
+$ ./bin/xinfra-monitor-start.sh config/multi-cluster-monitor.yaml
 ```
 
 ### Run checkstyle on the java code

bin/windows/kafka-monitor-start.bat

@@ -1,5 +1,5 @@
 @echo off
-REM Copyright 2016 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
+REM Copyright 2022 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
 REM file except in compliance with the License. You may obtain a copy of the License at
 REM
 REM    http://www.apache.org/licenses/LICENSE-2.0
@@ -15,7 +15,7 @@ popd
 
 
 IF [%1] EQU [] (
-	echo USAGE: %0 config/xinfra-monitor.properties
+	echo USAGE: %0 config/xinfra-monitor.yaml
 	EXIT /B 1
 )
 

bin/windows/kmf-run-class.bat

@@ -1,5 +1,5 @@
 @echo off
-REM Copyright 2016 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
+REM Copyright 2022 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
 REM file except in compliance with the License. You may obtain a copy of the License at
 REM
 REM    http://www.apache.org/licenses/LICENSE-2.0
@@ -10,12 +10,12 @@ REM an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expre
 setlocal enabledelayedexpansion
 
 IF [%1] EQU [] (
-	echo USAGE: %0 com.linkedin.xinfra.monitor.XinfraMonitor config/xinfra-monitor.properties
+	echo USAGE: %0 com.linkedin.xinfra.monitor.XinfraMonitor config/xinfra-monitor.yaml
 	EXIT /B 1
 )
 
 IF [%2] EQU [] (
-	echo USAGE: %0 %1 config/xinfra-monitor.properties
+	echo USAGE: %0 %1 config/xinfra-monitor.yaml
 	EXIT /B 1
 )
 

build.gradle

@@ -45,6 +45,7 @@ allprojects {
     compile group: 'org.apache.kafka', name: 'kafka-clients', version: '2.3.1'
     compile 'org.apache.commons:commons-lang3:3.12.0'
     compile 'com.linkedin.avroutil1:helper-all:0.2.81'
+    compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.10.3'
     testCompile 'org.mockito:mockito-core:2.24.0'
     testCompile 'org.testng:testng:6.8.8'
   }

config/multi-cluster-monitor.yaml

@@ -0,0 +1,80 @@
+---
+# Copyright 2022 LinkedIn Corp. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+# This properties file specifies an example configure to monitor a pipeline of Kafka clusters.
+# Users will need to update `zookeeper.connect` and `bootstrap.servers` to point to respective clusters.
+# More clusters can be added in the map for "topic.management.config.per.cluster" to reference
+# each cluster in the pipeline. The "produce.service.props" should use the first cluster and
+# the "consume.service.props" should use the last cluster in the pipeline.
+
+# Produce service: Configure Produce Service to produce to the first cluster of the pipeline
+# Consume service: Configure Consume Service to consume from the last cluster of the pipeline
+# Last cluster: If there are more than two clusters in the pipeline, add one property map for each one of them.
+#
+# For additional service configuration examples see `config/xinfra-monitor.yaml`
+
+multi-cluster-monitor:
+  class.name: com.linkedin.kmf.apps.MultiClusterMonitor
+  topic: kafka-monitor-topic
+
+  produce.service.props:
+    zookeeper.connect: localhost:2181/first_cluster
+    bootstrap.servers: localhost:9092
+    produce.record.delay.ms: 100
+    produce.producer.props:
+      client.id: kafka-monitor-client-id
+
+  consume.service.props:
+    zookeeper.connect: localhost:2181/last_cluster
+    bootstrap.servers: localhost:9095
+    consume.latency.sla.ms: '20000'
+    consume.consumer.props:
+      group.id: kafka-monitor-group-id
+  topic.management.props.per.cluster:
+    first-cluster:
+      bootstrap.servers: localhost:9092
+      zookeeper.connect: localhost:2181/first_cluster
+      topic-management.topicCreationEnabled: true
+      topic-management.replicationFactor: 1
+      topic-management.partitionsToBrokersRatio: 2
+      topic-management.rebalance.interval.ms: 600000
+      topic-management.topicFactory.props: {}
+    last-cluster:
+      bootstrap.servers: localhost:9095
+      zookeeper.connect: localhost:2181/last_cluster
+      topic-management.topicCreationEnabled: true
+      topic-management.replicationFactor: 1
+      topic-management.partitionsToBrokersRatio: 2
+      topic-management.rebalance.interval.ms: 600000
+      topic-management.topicFactory.props: {}
+
+reporter-service:
+  class.name: com.linkedin.kmf.services.DefaultMetricsReporterService
+  report.interval.sec: 1
+  report.metrics.list:
+    - kmf.services:type=produce-service,name=*:produce-availability-avg
+    - kmf.services:type=consume-service,name=*:consume-availability-avg
+    - kmf.services:type=produce-service,name=*:records-produced-total
+    - kmf.services:type=consume-service,name=*:records-consumed-total
+    - kmf.services:type=consume-service,name=*:records-lost-total
+    - kmf.services:type=consume-service,name=*:records-lost-rate
+    - kmf.services:type=consume-service,name=*:records-duplicated-total
+    - kmf.services:type=consume-service,name=*:records-delay-ms-avg
+    - kmf.services:type=produce-service,name=*:records-produced-rate
+    - kmf.services:type=produce-service,name=*:produce-error-rate
+    - kmf.services:type=consume-service,name=*:consume-error-rate
+    - kmf.services:type=commit-availability-service,name=*:offsets-committed-avg
+    - kmf.services:type=commit-availability-service,name=*:commit-latency-avg
+    - kmf.services:type=commit-availability-service,name=*:commit-availability-avg
+    - kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-avg
+    - kmf.services:type=commit-availability-service,name=*:offsets-committed-total
+    - kmf.services:type=commit-availability-service,name=*:failed-commit-offsets-total
+    -
+jolokia-service:
+  class.name: com.linkedin.kmf.services.JolokiaService