Enforce tests execute ok (rscada#165)

fredrik-sk · Jul 7, 2020 · 62b5829 · 62b5829
1 parent 62c1286
commit 62b5829
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 1 deletion.
diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml
@@ -12,7 +12,10 @@ jobs:
       run: rm -rf build || true && mkdir build && cd build && cmake .. -DLIBMBUS_BUILD_EXAMPLES=ON -DLIBMBUS_BUILD_TESTS=ON -DLIBMBUS_ENABLE_COVERAGE=ON && cmake --build . -j && cd ..
 
     - name: generate test frames
-      run: ./test/generate-xml.sh test/test-frames
+      run: |
+        ./test/generate-xml.sh test/test-frames
+        ./test/generate-xml.sh test/error-frames
+        ./test/generate-xml.sh test/unsupported-frames
 
     - name: install and run gcovr
       run: sudo pip install gcovr && gcovr build/.

diff --git a/test/generate-xml.sh b/test/generate-xml.sh
@@ -12,6 +12,12 @@
 #
 #------------------------------------------------------------------------------
 
+NUMBER_OF_PARSING_ERRORS=0
+FAILING_TESTS=failing_tests.txt
+NEW_TESTS=new_tests.txt
+touch $FAILING_TESTS
+touch $NEW_TESTS
+
 # Check commandline parameter
 if [ $# -ne 1 ]; then
     echo "usage: $0 path_to_directory_with_xml_files"
@@ -64,6 +70,7 @@ generate_xml() {
 
     # Check parsing result
     if [ $result -ne 0 ]; then
+        NUMBER_OF_PARSING_ERRORS=$((NUMBER_OF_PARSING_ERRORS + 1))
         echo "Unable to generate XML for $hexfile"
         rm "$directory/$filename$mode.xml.new"
         return 1
@@ -81,14 +88,17 @@ generate_xml() {
              ;;
         1)
              # different -> print diff
+             echo "== $directory/$filename$mode failed"
              cat "$directory/$filename$mode.dif" && rm "$directory/$filename$mode.dif"
              echo ""
+             echo "$filename$mode" >> $FAILING_TESTS
              ;;
         *)
              # no old -> rename XML
              echo "Create $filename$mode.xml"
              mv "$directory/$filename$mode.xml.new" "$directory/$filename$mode.xml"
              rm "$directory/$filename$mode.dif"
+             echo "$filename$mode" >> $NEW_TESTS
              ;;
     esac
 
@@ -105,3 +115,33 @@ for hexfile in "$directory"/*.hex;  do
     generate_xml "$directory" "$hexfile" "normalized"
 done
 
+# Check the size of the file $FAILING_TESTS. Make sure to indicate failure.
+if [ -s $FAILING_TESTS ]; then
+    echo "** There were errors in the following file(s):"
+    cat $FAILING_TESTS
+    exit 1
+else
+    rm $FAILING_TESTS
+fi
+if [ -s $NEW_TESTS ]; then
+    echo "** There were new test in the following file(s):"
+    cat $NEW_TESTS
+else
+    rm $FAILING_TESTS
+fi
+
+# Check that there was exactly the correct number of files that failed to parse
+EXPECTED_PARSE_ERRORS=0
+DIRECTORY_BASENAME="$(basename "$directory")"
+if [ "error-frames" = "$DIRECTORY_BASENAME" ]; then
+    EXPECTED_PARSE_ERRORS=30
+elif [ "unsupported-frames" = "$DIRECTORY_BASENAME" ]; then
+    EXPECTED_PARSE_ERRORS=12
+fi
+if [ $NUMBER_OF_PARSING_ERRORS -ne $EXPECTED_PARSE_ERRORS ]; then
+    echo "** There were $NUMBER_OF_PARSING_ERRORS files that did not parse, expected $EXPECTED_PARSE_ERRORS files."
+    echo
+    exit 1
+fi
+echo "** Tests executed successfully in \"$DIRECTORY_BASENAME\"."
+echo