Merge pull request #31 from CDOT-CV/develop

Standardizing Logging Level Strings, Fixed `do_kafka_test.sh` script

.devcontainer/devcontainer.json

@@ -19,6 +19,13 @@
 		"ms-vscode.cmake-tools"
 	],
 
+	"features": {
+		"docker-from-docker": {
+			"version": "latest",
+			"moby": true
+		}
+	},
+
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 

Dockerfile

@@ -3,6 +3,8 @@ USER root
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib
 ADD ./include /cvdi-stream/include
 ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

Dockerfile-nsv

@@ -5,6 +5,8 @@ ARG PPM_MAP_FILE
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -30,6 +32,7 @@ ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
 ADD config/${PPM_CONFIG_FILE} /cvdi-stream/config/
 ADD data/${PPM_MAP_FILE} /cvdi-stream/config/
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

Dockerfile.standalone

@@ -3,6 +3,8 @@ USER root
 
 WORKDIR /cvdi-stream
 
+ENV DEBIAN_FRONTEND=noninteractive
+
 # Add build tools.
 RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
 
@@ -26,6 +28,7 @@ ADD ./cv-lib /cvdi-stream/cv-lib
 ADD ./include /cvdi-stream/include
 ADD ./kafka-test /cvdi-stream/kafka-test
 ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
 
 # Do the build.
 RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make

Dockerfile.testing

@@ -0,0 +1,38 @@
+FROM ubuntu:18.04
+USER root
+
+WORKDIR /cvdi-stream
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Add build tools.
+RUN apt-get update && apt-get install -y software-properties-common wget git make gcc-7 g++-7 gcc-7-base && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 100
+
+# Install cmake.
+RUN apt install -y libprotobuf-dev protobuf-compiler
+RUN apt install -y cmake
+
+# Install librdkafka.
+RUN apt-get install -y libsasl2-dev libsasl2-modules libssl-dev librdkafka-dev
+
+# install python for testing
+RUN apt-get install -y python3
+
+# add the source and build files
+ADD CMakeLists.txt /cvdi-stream
+ADD ./src /cvdi-stream/src
+ADD ./cv-lib /cvdi-stream/cv-lib
+ADD ./include /cvdi-stream/include
+ADD ./kafka-test /cvdi-stream/kafka-test
+ADD ./unit-test-data /cvdi-stream/unit-test-data
+ADD ./config /cvdi-stream/config
+
+# Do the build.
+RUN export LD_LIBRARY_PATH=/usr/local/lib && mkdir /cvdi-stream-build && cd /cvdi-stream-build && cmake /cvdi-stream && make
+
+# Add test data. This changes frequently so keep it low in the file.
+ADD ./docker-test /cvdi-stream/docker-test
+
+# Run the tool.
+RUN chmod 7777 /cvdi-stream/docker-test/ppm_no_map.sh
+CMD ["/cvdi-stream/docker-test/ppm_no_map.sh"]

README.md

@@ -185,6 +185,32 @@ If the RPM_DEBUG environment variable is set to true, debug messages will be log
 ## Build & Exec Script
 The [`build_and_exec.sh`](./build_and_exec.sh) script can be used to build a tagged image of the PPM, run the container & enter it with an interactive shell. This script can be used to test the PPM in a standalone environment.
 
+This script should be run outside of the dev container in an environment where Docker is available.
+
+It should be noted that this script needs to use the LF end-of-line sequence.
+
+## Kafka Test Script
+The [do_kafka_test.sh](./do_kafka_test.sh) script is designed to perform integration tests on a Kafka instance. To execute the tests, this script relies on the following scripts: standalone.sh, standalone_multi.sh, do_bsm_test.sh, and do_tim_test.sh.
+
+To ensure proper execution, it is recommended to run this script outside of the dev container where docker is available. This is because the script will spin up a standalone kafka instance and will not be able to access the docker daemon from within the dev container.
+
+It should be noted that this script and any dependent scripts need to use the LF end-of-line sequence. These include the following:
+- do_kafka_test.sh
+- standalone.sh
+- standalone_multi.sh
+- do_bsm_test.sh
+- do_tim_test.sh
+- test_in.py
+- test_out.py
+
+The DOCKER_HOST_IP environment variable must be set to the IP address of the host machine. This is required for the script to function properly. This can be set by using the following command:
+
+```
+export DOCKER_HOST_IP=$(ifconfig | zgrep -m 1 -oP '(?<=inet\s)\d+(\.\d+){3}')
+```
+
+WSL will sometimes hang while the script waits for kafka to create topics. The script should exit after a number of attempts, but if it does not, running `wsl --shutdown` in a windows command prompt and restarting the docker services is recommended.
+
 ## Some Notes
 - The tests for this project can be run after compilation by running the "ppm_tests" executable.
 - When manually compiling with WSL, librdkafka will sometimes not be recognized. This can be resolved by utilizing the provided dev environment.

do_kafka_test.sh

@@ -1,71 +1,172 @@
 #!/bin/bash
-./start_kafka.sh
 
-# Wait until Kafka creates our topics.
-while true; do
-    ntopics=$(docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 | wc -l)
+# This script tests the PPM against a kafka cluster. It sets up variables for container and input data
+# file names. It starts a Kafka container using another script and checks that required topics are created.
+# If the container or topics are missing, the script exits. It builds a Docker image using the current
+# directory and specified name/tag. It runs a series of tests using a script with different properties
+# and input data files, outputting results to the console. It stops the Kafka container after the tests
+# are completed. The script performs five steps: set up the test environment, wait for Kafka to create
+# topics, build the PPM image, run the tests, and clean up.
 
-    if [[ $ntopics == "4" ]]; then 
-        echo 'Found 4 topics:'
-        docker exec jpocvdp_kafka_1 /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1 2> /dev/null
-
-        break   
-    fi
-
-    sleep 1
-done
+CYAN='\033[0;36m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
 
+CURRENT_DIR_NAME=${PWD##*/}
+KAFKA_CONTAINER_NAME=$CURRENT_DIR_NAME-kafka-1
 MAP_FILE=data/I_80.edges
 BSM_DATA_FILE=data/I_80_test.json
 TIM_DATA_FILE=data/I_80_test_TIMS.json
+PPM_CONTAINER_NAME=test_ppm_instance
+PPM_IMAGE_TAG=do-kafka-test-ppm-image
+PPM_IMAGE_NAME=jpo-cvdp_ppm
+
+setup() {
+    if [ -z $DOCKER_HOST_IP ]; then
+        echo "DOCKER_HOST_IP is not set. Exiting."
+        exit 1
+    fi
+
+    # print setup info
+    echo "=== Setup Info ==="
+    echo "DOCKER_HOST_IP: $DOCKER_HOST_IP"
+    echo "KAFKA_CONTAINER_NAME: $KAFKA_CONTAINER_NAME"
+    echo "MAP_FILE: $MAP_FILE"
+    echo "BSM_DATA_FILE: $BSM_DATA_FILE"
+    echo "TIM_DATA_FILE: $TIM_DATA_FILE"
+    echo "PPM_CONTAINER_NAME: $PPM_CONTAINER_NAME"
+    echo "PPM_IMAGE_TAG: $PPM_IMAGE_TAG"
+    echo "PPM_IMAGE_NAME: $PPM_IMAGE_NAME"
+    echo "=================="
+
+    ./start_kafka.sh
+}
+
+waitForKafkaToCreateTopics() {
+    maxAttempts=100
+    attempts=0
+    while true; do
+        attempts=$((attempts+1))
+        if [ $(docker ps | grep $KAFKA_CONTAINER_NAME | wc -l) == "0" ]; then
+            echo "Kafka container '$KAFKA_CONTAINER_NAME' is not running. Exiting."
+            ./stop_kafka.sh
+            exit 1
+        fi
+
+        ltopics=$(docker exec -it $KAFKA_CONTAINER_NAME /opt/kafka/bin/kafka-topics.sh --list --zookeeper 172.17.0.1)
+        allTopicsCreated=true
+        if [ $(echo $ltopics | grep "topic.FilteredOdeBsmJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.FilteredOdeTimJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.OdeBsmJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        elif [ $(echo $ltopics | grep "topic.OdeTimJson" | wc -l) == "0" ]; then
+            allTopicsCreated=false
+        fi
+
+        if [ $allTopicsCreated == true ]; then
+            echo "Kafka has created all required topics"
+            break
+        fi
+
+        sleep 1
+
+        if [ $attempts -ge $maxAttempts ]; then
+            echo "Kafka has not created all required topics after $maxAttempts attempts. Exiting."
+            ./stop_kafka.sh
+            exit 1
+        fi
+    done
+}
+
+buildPPMImage() {
+    docker build . -t $PPM_IMAGE_NAME:$PPM_IMAGE_TAG -f Dockerfile.testing
+}
+
+run_tests() {
+    echo "--- File Being Used ---"
+    echo $MAP_FILE
+    echo $BSM_DATA_FILE
+    echo $TIM_DATA_FILE
+    echo "-----------------"
+
+    numberOfTests=10
+    echo -e $YELLOW"Test 1/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 2/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 3/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 4/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 5/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 6/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 7/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 8/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 9/$numberOfTests"$NC
+    ./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18
+    echo ""
+    echo ""
+
+    echo -e $YELLOW"Test 10/$numberOfTests (2 tests in one)"$NC
+    ./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23
+}
+
+cleanup() {
+    echo "[log] stopping Kafka"
+    ./stop_kafka.sh
+}
+
+run() {
+    numberOfSteps=5
+    echo ""
+    echo -e $CYAN"Step 1/$numberOfSteps: Set up test environment"$NC
+    setup
+
+    echo ""
+    echo -e $CYAN"Step 2/$numberOfSteps: Wait for Kafka to create topics"$NC
+    waitForKafkaToCreateTopics
+
+    echo ""
+    echo -e $CYAN"Step 3/$numberOfSteps: Build PPM image"$NC
+    buildPPMImage
+
+    echo ""
+    echo -e $CYAN"Step 4/$numberOfSteps: Run tests"$NC
+    run_tests
+
+    echo ""
+    echo -e $CYAN"Step 5/$numberOfSteps: Cleanup"$NC
+    cleanup
+}
 
-echo $MAP_FILE
-echo $BSM_DATA_FILE
-echo $TIM_DATA_FILE
-
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c1.properties $BSM_DATA_FILE BSM 0
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c2.properties $BSM_DATA_FILE BSM 10
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c3.properties $BSM_DATA_FILE BSM 18
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c4.properties $BSM_DATA_FILE BSM 23
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c5.properties $BSM_DATA_FILE BSM 33
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/bsm-test/c6.properties $BSM_DATA_FILE BSM 43
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c1.properties $TIM_DATA_FILE TIM 0
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c2.properties $TIM_DATA_FILE TIM 10
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone.sh $MAP_FILE config/tim-test/c3.properties $TIM_DATA_FILE TIM 18
-echo ""
-echo ""
-
-sleep 1
-./test-scripts/standalone_multi.sh $MAP_FILE config/bsm-test/c6.properties config/tim-test/c3.properties $BSM_DATA_FILE $TIM_DATA_FILE 48 23
+run

docker-compose-kafka.yml

@@ -0,0 +1,16 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP}
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "topic.OdeBsmJson:1:1,topic.FilteredOdeBsmJson:1:1,topic.OdeTimJson:1:1,topic.FilteredOdeTimJson:1:1"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock