OCPP E2E tests (#959)

Adding OCPP E2E tests into seperate testing directory ocpp_tests . The tests include tests for OCPP1.6 and OCPP2.0.1 for different feature profiles and functional blocks.
---------

Signed-off-by: Piet Gömpel <pietgoempel@gmail.com>
Signed-off-by: Kai-Uwe Hermann <kai-uwe.hermann@pionix.de>
Co-authored-by: Kai-Uwe Hermann <kai-uwe.hermann@pionix.de>

.ci/build-kit/scripts/create_ocpp_tests_image.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+rsync -a "$EXT_MOUNT/source/tests" ./
+retVal=$?
+
+if [ $retVal -ne 0 ]; then
+    echo "Failed to copy tests"
+    exit $retVal
+fi
+
+pip install --break-system-packages \
+    "$EXT_MOUNT"/wheels/everestpy-*.whl \
+    "$EXT_MOUNT"/wheels/everest_testing-*.whl \
+    "$EXT_MOUNT"/wheels/iso15118-*.whl \
+    pytest-html
+retVal=$?
+
+if [ $retVal -ne 0 ]; then
+    echo "Failed to pip-install"
+    exit $retVal
+fi
+
+pip install --break-system-packages -r tests/ocpp_tests/requirements.txt
+
+$(cd ./tests/ocpp_tests/test_sets/everest-aux/ && ./install_certs.sh "$EXT_MOUNT/dist" && ./install_configs.sh "$EXT_MOUNT/dist")

.ci/e2e/scripts/run_ocpp_tests.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+cd tests
+
+PARALLEL_TESTS=$(nproc)
+
+echo "Running $PARALLEL_TESTS ocpp tests in parallel"
+
+pytest \
+  -rA \
+  -d --tx "$PARALLEL_TESTS"*popen//python=python3 \
+  --max-worker-restart=0 \
+  --timeout=300 \
+  --junitxml="$EXT_MOUNT/ocpp-tests-result.xml" \
+  --html="$EXT_MOUNT/ocpp-tests-report.html" \
+  --self-contained-html \
+  ocpp_tests/test_sets/ocpp16/*.py \
+  ocpp_tests/test_sets/ocpp201/*.py \
+  --everest-prefix "$EXT_MOUNT/dist"
+retVal=$?
+
+if [ $retVal -ne 0 ]; then
+    echo "OCPP tests failed with return code $retVal"
+    exit $retVal
+fi

.github/workflows/build_and_test.yaml

@@ -29,3 +29,91 @@ jobs:
       do_not_run_coverage_badge_creation: true
       run_install_wheels: true
       run_integration_tests: true
+  ocpp-tests:
+    name: OCPP Tests
+    needs:
+      - ci
+    runs-on: ${{ inputs.runner || 'ubuntu-22.04' }}
+    steps:
+      - name: Download dist dir
+        uses: actions/download-artifact@v4.1.8
+        with:
+          name: dist
+      - name: Extract dist.tar.gz
+        run: |
+          tar -xzf ${{ github.workspace }}/dist.tar.gz -C ${{ github.workspace }}
+      - name: Download wheels
+        # if: ${{ inputs.run_install_wheels == 'true' }}
+        uses: actions/download-artifact@v4.1.8
+        with:
+          name: wheels
+          path: wheels
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.2
+        with:
+          path: source
+      - name: Setup run scripts
+        run: |
+          mkdir scripts
+          rsync -a source/.ci/build-kit/scripts/ scripts
+      - name: Docker Meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.BUILD_KIT_IMAGE_NAME }}
+      - name: Set output tag
+        id: buildkit_tag
+        shell: python3 {0}
+        run: |
+          import os
+          tags = "${{ steps.meta.outputs.tags }}".split(",")
+          if len(tags) == 0:
+            print("No tags found!❌")
+            exit(1)
+          tag = f"local/build-kit-everest-core:{tags[0]}"
+          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+            f.write(f"tag={tag}\n")
+            print(f"Set tag={tag}")
+      - name: Download build-kit image
+        uses: actions/download-artifact@v4
+        with:
+          name: build-kit
+      - name: Load build-kit image
+        run: |
+          docker load -i build-kit.tar
+          docker image tag ${{ steps.buildkit_tag.outputs.tag }} build-kit
+      - name: Create integration-image
+        run: |
+          docker run \
+          --volume "${{ github.workspace }}:/ext" \
+          --name integration-container \
+          build-kit run-script create_ocpp_tests_image
+          docker commit integration-container integration-image
+      - name: Run OCPP tests
+        id: run_ocpp_tests
+        continue-on-error: true
+        run: |
+          docker compose \
+          -f source/.ci/e2e/docker-compose.yaml \
+          run \
+          e2e-test-server \
+          run-script run_ocpp_tests
+      - name: Upload result and report as artifact
+        continue-on-error: true
+        if: ${{ steps.run_ocpp_tests.outcome == 'success' || steps.run_ocpp_tests.outcome == 'failure' }}
+        uses: actions/upload-artifact@v4.4.3
+        with:
+          if-no-files-found: error
+          name: ocpp-tests-report
+          path: |
+            ocpp-tests-result.xml
+            ocpp-tests-report.html
+      - name: Render OCPP tests result
+        if: ${{ steps.run_ocpp_tests.outcome == 'success' || steps.run_ocpp_tests.outcome == 'failure' }}
+        uses: pmeier/pytest-results-action@v0.7.1
+        with:
+          path: ocpp-tests-result.xml
+          summary: True
+          display-options: fEX
+          fail-on-empty: True
+          title: Test results

CMakeLists.txt

@@ -169,7 +169,7 @@ add_custom_target(install_everest_testing
         if [ -z "${CPM_PACKAGE_everest-utils_SOURCE_DIR}" ] \;
         then echo "Could not determine location of everest-utils, please install everest-testing manually!" \;
         else echo "Found everest-utils at ${CPM_PACKAGE_everest-utils_SOURCE_DIR}" \;
-             ${PYTHON_EXECUTABLE} -m pip install "${CPM_PACKAGE_everest-utils_SOURCE_DIR}/everest-testing" \;
+             ${Python3_EXECUTABLE} -m pip install -e "${CPM_PACKAGE_everest-utils_SOURCE_DIR}/everest-testing" \;
         fi\;
     DEPENDS
         everestpy_pip_install_dist

dependencies.yaml

@@ -86,7 +86,7 @@ ext-mbedtls:
 # everest-testing and ev-dev-tools
 everest-utils:
   git: https://github.com/EVerest/everest-utils.git
-  git_tag: v0.4.2
+  git_tag: v0.4.3
 
 # unit testing
 gtest:

modules/System/signed_firmware_downloader.sh

@@ -2,7 +2,7 @@
 
 . "${1}"
 
-mkdir /tmp/signature_validation
+SIGNATURE_VALIDATION_DIR=$(mktemp -d /tmp/signature_validation_XXXXX)
 sleep 2
 echo "$DOWNLOADING"
 
@@ -12,11 +12,11 @@ curl_exit_code=$?
 sleep 2
 if [[ $curl_exit_code -eq 0 ]]; then
     echo "$DOWNLOADED"
-    echo -e "${4}" >/tmp/signature_validation/firmware_signature.base64
-    echo -e "${5}" >/tmp/signature_validation/firmware_cert.pem
-    openssl x509 -pubkey -noout -in /tmp/signature_validation/firmware_cert.pem >/tmp/signature_validation/pubkey.pem
-    openssl base64 -d -in /tmp/signature_validation/firmware_signature.base64 -out /tmp/signature_validation/firmware_signature.sha256
-    r=$(openssl dgst -sha256 -verify /tmp/signature_validation/pubkey.pem -signature /tmp/signature_validation/firmware_signature.sha256 "${3}")
+    echo -e "${4}" >"$SIGNATURE_VALIDATION_DIR/firmware_signature.base64"
+    echo -e "${5}" >"$SIGNATURE_VALIDATION_DIR/firmware_cert.pem"
+    openssl x509 -pubkey -noout -in "$SIGNATURE_VALIDATION_DIR/firmware_cert.pem" >"$SIGNATURE_VALIDATION_DIR/pubkey.pem"
+    openssl base64 -d -in "$SIGNATURE_VALIDATION_DIR/firmware_signature.base64" -out "$SIGNATURE_VALIDATION_DIR/firmware_signature.sha256"
+    r=$(openssl dgst -sha256 -verify "$SIGNATURE_VALIDATION_DIR/pubkey.pem" -signature "$SIGNATURE_VALIDATION_DIR/firmware_signature.sha256" "${3}")
 
     if [ "$r" = "Verified OK" ]; then
         echo "$SIGNATURE_VERIFIED"
@@ -27,4 +27,4 @@ else
     echo "$DOWNLOAD_FAILED"
 fi
 
-rm -rf /tmp/signature_validation
+rm -rf "$SIGNATURE_VALIDATION_DIR"

tests/ocpp_tests/.gitignore

@@ -0,0 +1,10 @@
+build
+__pycache__
+*.egg-info
+.pytest_cache
+.venv
+results.xml
+result.xml
+report.html
+**/.DS_Store
+**/.idea

tests/ocpp_tests/README.md

@@ -0,0 +1,69 @@
+# OCPP Integration Tests
+
+This directory contains some test tooling and integration tests
+for OCPP1.6 and OCPP2.0.1.
+
+## Requirements
+
+In order to run the integration tests, you need to have everest-core compiled
+and installed on your system. Please also make sure to install the python
+requirements.
+
+```bash
+cd everest-core/
+cmake -S . -B build -DBUILD_TESTING=ON
+cmake --build build --target install --parallel -j$(nproc)
+. build/venv/bin/activate
+cmake --build build --target everestpy_pip_install_dist
+cmake --build build --target everest-testing_pip_install_dist
+cmake --build build --target iso15118_pip_install_dist
+python3 -m pip install aiofile>=3.7.4
+python3 -m pip install netifaces>=0.11.0
+cd tests/ocpp_tests
+python3 -m pip install -r requirements.txt
+```
+
+## Run the tests
+
+You can run the integration tests using the convenience scripts
+provided in this directory e.g.
+
+```bash
+./run-testing.sh
+```
+
+This command runs all test cases in parallel.
+The time for running the test cases depends on your system.
+It usually takes a couple of minutes.
+You can check out the test results by opening the generated `results.html`.
+
+You can choose to run the tests sequentially and/or only run subsets
+for OCPP1.6 or OCPP2.0.1 using any of the other run scripts.
+
+Alternatively, you can run individual test sets or test cases using
+
+```bash
+python3 -m pytest test_sets/ocpp201/remote_control.py \
+  --everest-prefix <path-to-everest-core-installation-directory> \
+  -k 'test_F01_F02_F03'
+```
+
+e.g.
+
+```bash
+python3 -m pytest test_sets/ocpp201/remote_control.py \
+  --everest-prefix ~/checkout/everest-core/build/dist \
+  -k 'test_F01_F02_F03'
+```
+
+This runs test case `test_F01_F02_F03`
+specified in `test_sets/ocpp201/remote_control.py`.
+
+If you run the test cases individually,
+make sure to have all required certificates and configs
+for the test cases installed using the
+convenience scripts inside [test_sets/everest-aux](test_sets/everest-aux/)
+
+```bash
+./install_certs <path-to-everest-core-installation-directory>
+./install_configs <path-to-everest-core-installation-directory>