Re-enable TC-RR-1.1 in CI (project-chip#22292)

* Re-enable TC-RR-1.1 in CI

- A regression had occured on RR-1.1 in master before CI was attempted
  in project-chip#22143 which prevented the CI test to be enabled

This PR:

- Fixes TC-RR-1.1
- Enables TC-RR-1.1, TC-SC-3.6 and TC-DA-1.7 to keep them fresh and
  improve integration test coverage (TC-RR-1.1 tests a lot)
- Improve run_python_test.py to do better factory reset

Fixes project-chip#21736

Testing done:
- Manually tested the fix of TC-RR-1.1 on ESP32 and Linux
- CI tests of the above is automated

* Attempt to pass CI

* Debug CI

* More debug of CI

* More CI fixes

* Fix more CI

.github/workflows/tests.yaml

@@ -243,7 +243,7 @@ jobs:
  container:
  image: connectedhomeip/chip-build:0.5.91
  options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
- net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"
+ net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0"
 
  steps:
  - name: Checkout
@@ -273,19 +273,18 @@ jobs:
  scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
  ./scripts/run_in_build_env.sh \
  "./scripts/build/build_examples.py \
- --target linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test \
+ --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test \
  --target linux-x64-python-bindings \
  build \
  --copy-artifacts-to objdir-clone \
  "
  - name: Run Tests
- timeout-minutes: 40
+ timeout-minutes: 10
  run: |
- scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
- # the below tests are broken, enable them when they are fixed. Suspect hermetic issues for now
- # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
- # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--commissioning-method on-network --discriminator 1234 --passcode 20202021"'
- # scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py"'
+ scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --script-args "--log-level INFO -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout"'
+ scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_RR_1_1.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
+ scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_SC_3_6.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021"'
+ scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app --factoryreset --app-args "--discriminator 1234 --KVS kvs1 --trace_decode 1" --script "src/python_testing/TC_DA_1_7.py" --script-args "--storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --bool-arg allow_sdk_dac:true"'
  - name: Uploading core files
  uses: actions/upload-artifact@v2
  if: ${{ failure() && !env.ACT }}

scripts/tests/run_python_test.py

@@ -22,6 +22,7 @@
 import pathlib
 import pty
 import queue
+import re
 import shlex
 import signal
 import subprocess
@@ -76,10 +77,32 @@ def DumpProgramOutputToQueue(thread_list: typing.List[threading.Thread], tag: st
 @click.option("--script-gdb", is_flag=True, help='Run script through gdb')
 def main(app: str, factoryreset: bool, app_args: str, script: str, script_args: str, script_gdb: bool):
  if factoryreset:
+ # Remove native app config
  retcode = subprocess.call("rm -rf /tmp/chip* /tmp/repl*", shell=True)
  if retcode != 0:
  raise Exception("Failed to remove /tmp/chip* for factory reset.")
 
+ print("Contents of test directory: %s" % os.getcwd())
+ print(subprocess.check_output(["ls -l"], shell=True).decode('us-ascii'))
+
+ # Remove native app KVS if that was used
+ kvs_match = re.search(r"--KVS (?P<kvs_path>[^ ]+)", app_args)
+ if kvs_match:
+ kvs_path_to_remove = kvs_match.group("kvs_path")
+ retcode = subprocess.call("rm -f %s" % kvs_path_to_remove, shell=True)
+ print("Trying to remove KVS path %s" % kvs_path_to_remove)
+ if retcode != 0:
+ raise Exception("Failed to remove %s for factory reset." % kvs_path_to_remove)
+
+ # Remove Python test admin storage if provided
+ storage_match = re.search(r"--storage-path (?P<storage_path>[^ ]+)", script_args)
+ if storage_match:
+ storage_path_to_remove = storage_match.group("storage_path")
+ retcode = subprocess.call("rm -f %s" % storage_path_to_remove, shell=True)
+ print("Trying to remove storage path %s" % storage_path_to_remove)
+ if retcode != 0:
+ raise Exception("Failed to remove %s for factory reset." % storage_path_to_remove)
+
  coloredlogs.install(level='INFO')
 
  log_queue = queue.Queue()

src/python_testing/matter_testing_support.py

@@ -686,7 +686,7 @@ def default_matter_test_main(argv=None, **kwargs):
  matter_test_config = parse_matter_test_args(argv)
 
  # Allow override of command line from optional arguments
- if matter_test_config.controller_cat_tags is None and "controller_cat_tags" in kwargs:
+ if not matter_test_config.controller_cat_tags and "controller_cat_tags" in kwargs:
  matter_test_config.controller_cat_tags = kwargs["controller_cat_tags"]
 
  # Find the test class in the test script.