Fixed do_kafka_test.sh script

.devcontainer/devcontainer.json

@@ -19,6 +19,13 @@
 		"ms-vscode.cmake-tools"
 	],
 
+	"features": {
+		"docker-from-docker": {
+			"version": "latest",
+			"moby": true
+		}
+	},
+
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 

Dockerfile

@@ -13,6 +13,9 @@ RUN apt install -y cmake
 # Install librdkafka.
 RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
+# install python for testing
+RUN apt-get install -y python3
+
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
 ADD ./src /cvdi-stream/src

Dockerfile-nsv

@@ -15,6 +15,9 @@ RUN apt install -y cmake
 # Install librdkafka.
 RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
+# install python for testing
+RUN apt-get install -y python3
+
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
 ADD ./src /cvdi-stream/src

Dockerfile.standalone

@@ -13,6 +13,9 @@ RUN apt install -y cmake
 # Install librdkafka.
 RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
 
+# install python for testing
+RUN apt-get install -y python3
+
 # add the source and build files
 ADD CMakeLists.txt /cvdi-stream
 ADD ./src /cvdi-stream/src

do_kafka_test.sh

@@ -1,71 +1,150 @@
 #!/bin/bash
-./start_kafka.sh
 
-# Wait until Kafka creates our topics.
-while true; do
-    ntopics=$(docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l)
+# This script tests the PPM against a kafka cluster. It sets up variables for container and input data
+# file names. It starts a Kafka container using another script and checks that required topics are created.
+# If the container or topics are missing, the script exits. It builds a Docker image using the current
+# directory and specified name/tag. It runs a series of tests using a script with different properties
+# and input data files, outputting results to the console. It stops the Kafka container after the tests
+# are completed. The script performs five steps: set up the test environment, wait for Kafka to create
+# topics, build the PPM image, run the tests, and clean up.
 
-    if [[ $ntopics == "4" ]]; then 
-        echo 'Found 4 topics:'
-        docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null
-
-        break   
-    fi
-
-    sleep 1
-done
+CYAN='\033[0;36m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
 
+KAFKA_CONTAINER_NAME=jpo-cvdp-kafka-1
 MAP_FILE=data/I_80.edges
 BSM_DATA_FILE=data/I_80_test.json
 TIM_DATA_FILE=data/I_80_test_TIMS.json
+PPM_CONTAINER_NAME=ppm_kafka
+PPM_IMAGE_TAG=do-kafka-test-ppm-image
+PPM_IMAGE_NAME=jpo-cvdp_ppm
+
+setup() {
+    if [ -z $DOCKER_HOST_IP ]; then
+        echo "DOCKER_HOST_IP is not set. Exiting."
+        exit 1
+    fi
+
+    ./start_kafka.sh
+}
+
+waitForKafkaToCreateTopics() {
+    while true; do
+        if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then
+            echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting."
+            ./stop_kafka.sh
+            exit 1
+        fi
+
+        ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1)
+        allTopicsCreated=true
+        if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.FilteredOdeTimJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.OdeBsmJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.OdeTimJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        fi
+
+        if [ $allTopicsCreated == true ]; then
+            echo "Kafka has created all required topics"
+            break
+        fi
+
+        sleep 1
+    done
+}
+
+buildPPMImage() {
+    docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG
+}
+
+run_tests() {
+    echo "--- File Being Used ---"
+    echo $MAP_FILE
+    echo $BSM_DATA_FILE
+    echo $TIM_DATA_FILE
+    echo "-----------------"
+
+    numberOfTests=10
+    echo -e $YELLOW"Test 1/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 2/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 3/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 4/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 5/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 6/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 7/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 8/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 9/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 10/$numberOfTests (2 tests in one)"$NC
+    ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23
+}
+
+cleanup() {
+    echo "[log] stopping Kafka"
+    ./stop_kafka.sh
+}
+
+run() {
+    numberOfSteps=5
+    echo ""
+    echo -e $CYAN"Step 1/$numberOfSteps: Set up test environment"$NC
+    setup
+
+    echo ""
+    echo -e $CYAN"Step 2/$numberOfSteps: Wait for Kafka to create topics"$NC
+    waitForKafkaToCreateTopics
+
+    echo ""
+    echo -e $CYAN"Step 3/$numberOfSteps: Build PPM image"$NC
+    buildPPMImage
+
+    echo ""
+    echo -e $CYAN"Step 4/$numberOfSteps: Run tests"$NC
+    run_tests
+
+    echo ""
+    echo -e $CYAN"Step 5/$numberOfSteps: Cleanup"$NC
+    cleanup
+}
 
-echo $MAP_FILE
-echo $BSM_DATA_FILE
-echo $TIM_DATA_FILE
-
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23
+run

docker-compose-kafka.yml

@@ -0,0 +1,16 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

docker-test/do_bsm_test.sh

@@ -1,7 +1,18 @@
 #!/bin/bash
+
+# This script produces and consumes messages from Kafka topics. It reads a JSON file containing raw
+# BSM data, processes it with a Python script, and then sends the output to a Kafka topic. Then it
+# consumes the filtered messages from the Kafka topic, using a specified offset, and checks if
+# any messages were received. If no messages were received after a certain number of attempts, the
+# script exits with an error message. Otherwise, the script exits with a success message.
+
 export LD_LIBRARY_PATH=/usr/local/lib
 
-broker=172.17.0.1:9092
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+broker=$DOCKER_HOST_IP:9092
 
 echo "**************************"
 echo "Producing Raw BSMs..."
@@ -15,7 +26,11 @@ echo "**************************"
 echo "Consuming Filtered BSMs at offset "$offset "..." 
 echo "**************************"
 
+attempts=0
+max_attempts=5
 while true; do
+    attempts=$((attempts+1))
+
     /cvdi-stream-build/kafka-test/kafka_tool -C -b $broker -p 0 -t topic.FilteredOdeBsmJson -e -o $offset 2> con.err | /cvdi-stream/docker-test/test_out.py > tmp.out
 
     lines=$(cat tmp.out | wc -l)
@@ -24,5 +39,16 @@ while true; do
         cat tmp.out
 
         break
+    else
+        if [[ $attempts > $max_attempts ]]; then
+            echo "No data received after $max_attempts attempts. Exiting..."
+            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+            echo -e $RED"TEST FAILED!"$NC
+            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+            exit 1
+        fi
     fi
 done
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+echo -e $GREEN"TEST PASSED!"$NC
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"

docker-test/do_tim_test.sh

@@ -1,7 +1,18 @@
 #!/bin/bash
+
+# This script produces and consumes messages from Kafka topics. It reads a JSON file containing raw
+# TIM data, processes it with a Python script, and then sends the output to a Kafka topic. Then it
+# consumes the filtered messages from the Kafka topic, using a specified offset, and checks if
+# any messages were received. If no messages were received after a certain number of attempts, the
+# script exits with an error message. Otherwise, the script exits with a success message.
+
 export LD_LIBRARY_PATH=/usr/local/lib
 
-broker=172.17.0.1:9092
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+broker=$DOCKER_HOST_IP:9092
 
 echo "**************************"
 echo "Producing Raw TIMs..."
@@ -15,7 +26,11 @@ echo "**************************"
 echo "Consuming Filtered TIMs at offset "$offset "..." 
 echo "**************************"
 
+attempts=0
+max_attempts=5
 while true; do
+    attempts=$((attempts+1))
+
     /cvdi-stream-build/kafka-test/kafka_tool -C -b $broker -p 0 -t topic.FilteredOdeTimJson -e -o $offset 2> con.err | /cvdi-stream/docker-test/test_out.py > tmp.out
 
     lines=$(cat tmp.out | wc -l)
@@ -24,5 +39,16 @@ while true; do
         cat tmp.out
 
         break
+    else
+        if [[ $attempts > $max_attempts ]]; then
+            echo "No data received after $max_attempts attempts. Exiting..."
+            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+            echo -e $RED"TEST FAILED!"$NC
+            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+            exit 1
+        fi
     fi
 done
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"
+echo -e $GREEN"TEST PASSED!"$NC
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~"

docker-test/ppm_standalone.sh

@@ -1,7 +1,14 @@
 #!/bin/sh
+
+# This script starts the PPM tool with the specified configuration file, using the DOCKER_HOST_IP
+# environment variable to connect to a Kafka broker.
+
 export LD_LIBRARY_PATH=/usr/local/lib
 
-broker=172.17.0.1:9092
+if [ -z "$DOCKER_HOST_IP" ]; then
+    echo "DOCKER_HOST_IP is not set. Exiting."
+    exit 1
+fi
 
 # Start the DI tool.
-/cvdi-stream-build/ppm -c /ppm_data/config.properties -o end
+/cvdi-stream-build/ppm -c /ppm_data/config.properties -b ${DOCKER_HOST_IP}:9092 -o end