fail when new errors appear

.github/workflows/sanity-checks.yml

@@ -27,27 +27,26 @@ jobs:
       - name: Install ansible-base ${{ matrix.ansible }}
         run: pip install https://github.com/ansible/ansible/archive/${{ matrix.ansible }}.tar.gz --disable-pip-version-check
 
-      # Always succeeding sanity test until all errors are fixed
+      # Fail when new errors appear
       - name: Run sanity test
         run: |
-          sanity=$(\
-            ansible-test \
-            sanity \
-            --verbose \
-            --docker \
-            --color \
-            --coverage \
-            --failure-ok \
-            2>&1 \
-          )
-          echo "${sanity}"
-          echo errors=$(grep -c "ERROR: [rp]" <<< "${sanity}") >> $GITHUB_OUTPUT
+          set -ex
+          git checkout -b branch
+          git fetch --unshallow origin main
+          git config user.name 'Fake' && git config user.email 'fake@localhost'
+          git merge origin/main --no-edit
+          cmd="ansible-test sanity --verbose --color --docker --coverage --failure-ok"
+          $cmd 2>&1 | tee branch.output
+          git checkout main
+          $cmd > main.output 2>&1
+          for key in branch main; do
+            grep -E "(ERROR|FATAL):" "$key.output" | \
+            grep -v "issue(s) which need to be resolved\|See error output above for details." > "$key.errors"
+          done
+          set +x
+          echo "============================================================"
+          if diff -u main.errors branch.errors|grep '^+'; then
+             echo "Fix the regressions listed above (starting with a '+')"
+             exit 1
+          fi
         working-directory: ${{ matrix.ansible }}/ansible_collections/redhatci/ocp
-        id: sanity_test
-
-      - name: Error summary
-        run: |
-          echo "## Error summary" >> ${GITHUB_STEP_SUMMARY}
-          echo "" >> ${GITHUB_STEP_SUMMARY}
-          echo "Total Errors: ${{ steps.sanity_test.outputs.errors }}" >> ${GITHUB_STEP_SUMMARY}
-          echo "" >> ${GITHUB_STEP_SUMMARY}