Merge commit '08c95bf47ba6d0c5d11a7616e42e10b8dbde19eb'

.github/workflows/test.yaml

@@ -103,6 +103,20 @@ jobs:
         timeout-minutes: 5
         run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py"
 
+  mutation:
+    name: Mutation tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need master to get diff
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Mutation tests
+        timeout-minutes: 5
+        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh"
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-latest

.gitignore

@@ -29,3 +29,7 @@ nosetests.xml
 *.gcno
 tests/safety/coverage-out
 tests/safety/coverage.info
+
+*.profraw
+*.profdata
+mull.yml

Dockerfile

@@ -18,7 +18,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt clean && \
     cd /usr/lib/gcc/arm-none-eabi/* && \
     rm -rf arm/ && \
-    rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+    rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp && \
+    apt-get update && apt-get install -y clang-17 && \
+    ln -s $(which clang-17) /usr/bin/clang
+
+RUN apt-get update && apt-get install -y curl && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash && \
+    apt-get update && apt-get install -y mull-17
 
 ENV CPPCHECK_DIR=/tmp/cppcheck
 COPY tests/misra/install.sh /tmp/

Jenkinsfile

@@ -106,7 +106,7 @@ pipeline {
                 phone_steps("panda-tres", [
                   ["build", "scons -j4"],
                   ["flash", "cd tests/ && ./reflash_internal_panda.py"],
-                  ["flash jungle", "cd board/jungle && ./flash.py"],
+                  ["flash jungle", "cd board/jungle && ./flash.py --all"],
                   ["test", "cd tests/hitl && HW_TYPES=9 pytest -n0 --durations=0 2*.py [5-9]*.py"],
                 ])
               }
@@ -118,7 +118,7 @@ pipeline {
                 phone_steps("panda-dos", [
                   ["build", "scons -j4"],
                   ["flash", "cd tests/ && ./reflash_internal_panda.py"],
-                  ["flash jungle", "cd board/jungle && ./flash.py"],
+                  ["flash jungle", "cd board/jungle && ./flash.py --all"],
                   ["test", "cd tests/hitl && HW_TYPES=6 pytest -n0 --durations=0 [2-9]*.py -k 'not test_send_recv'"],
                 ])
               }

README.md

@@ -49,7 +49,7 @@ In addition, we run the [ruff linter](https://github.com/astral-sh/ruff) and [my
 Setup dependencies:
 ```bash
 # Ubuntu
-sudo apt-get install dfu-util gcc-arm-none-eabi python3-pip libffi-dev git
+sudo apt-get install dfu-util gcc-arm-none-eabi python3-pip libffi-dev git clang-17
 
 # macOS
 brew install --cask gcc-arm-embedded

SConstruct

@@ -16,6 +16,10 @@ AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
 
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
+
 env = Environment(
   COMPILATIONDB_USE_ABSPATH=True,
   tools=["default", "compilation_db"],

board/flash.py

@@ -1,18 +1,27 @@
 #!/usr/bin/env python3
 import os
 import subprocess
+import argparse
 
 from panda import Panda
 
 board_path = os.path.dirname(os.path.realpath(__file__))
 
 if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--all", action="store_true", help="Recover all Panda devices")
+  args = parser.parse_args()
+
   subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) {board_path}", shell=True)
 
-  serials = Panda.list()
-  print(f"found {len(serials)} panda(s) - {serials}")
+  if args.all:
+    serials = Panda.list()
+    print(f"found {len(serials)} panda(s) - {serials}")
+  else:
+    serials = [None]
+
   for s in serials:
-    print("flashing", s)
     with Panda(serial=s) as p:
+      print("flashing", p.get_usb_serial())
       p.flash()
   exit(1 if len(serials) == 0 else 0)

board/jungle/flash.py

@@ -1,19 +1,27 @@
 #!/usr/bin/env python3
 import os
 import subprocess
+import argparse
 
 from panda import PandaJungle
 
 board_path = os.path.dirname(os.path.realpath(__file__))
 
 if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--all", action="store_true", help="Recover all panda jungle devices")
+  args = parser.parse_args()
+
   subprocess.check_call(f"scons -C {board_path}/.. -u -j$(nproc) {board_path}", shell=True)
 
-  serials = PandaJungle.list()
-  print(f"found {len(serials)} panda jungle(s) - {serials}")
+  if args.all:
+    serials = PandaJungle.list()
+    print(f"found {len(serials)} panda jungles(s) - {serials}")
+  else:
+    serials = [None]
+
   for s in serials:
-    print("flashing", s)
     with PandaJungle(serial=s) as p:
+      print("flashing", p.get_usb_serial())
       p.flash()
-
   exit(1 if len(serials) == 0 else 0)

board/jungle/recover.py

@@ -2,17 +2,23 @@
 import os
 import time
 import subprocess
+import argparse
 
 from panda import PandaJungle, PandaJungleDFU
 
 board_path = os.path.dirname(os.path.realpath(__file__))
 
 if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--all", action="store_true", help="Recover all panda jungle devices")
+  args = parser.parse_args()
+
   subprocess.check_call(f"scons -C {board_path}/.. -u -j$(nproc) {board_path}", shell=True)
 
-  for s in PandaJungle.list():
-    print("putting", s, "in DFU mode")
+  serials = PandaJungle.list() if args.all else [None]
+  for s in serials:
     with PandaJungle(serial=s) as p:
+      print(f"putting {p.get_usb_serial()} in DFU mode")
       p.reset(enter_bootstub=True)
       p.reset(enter_bootloader=True)
 

board/recover.py

@@ -2,17 +2,23 @@
 import os
 import time
 import subprocess
+import argparse
 
 from panda import Panda, PandaDFU
 
 board_path = os.path.dirname(os.path.realpath(__file__))
 
 if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--all", action="store_true", help="Recover all Panda devices")
+  args = parser.parse_args()
+
   subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) {board_path}", shell=True)
 
-  for s in Panda.list():
-    print("putting", s, "in DFU mode")
+  serials = Panda.list() if args.all else [None]
+  for s in serials:
     with Panda(serial=s) as p:
+      print(f"putting {p.get_usb_serial()} in DFU mode")
       p.reset(enter_bootstub=True)
       p.reset(enter_bootloader=True)
 

python/__init__.py

@@ -224,18 +224,46 @@ class Panda:
   FLAG_FORD_LONG_CONTROL = 1
   FLAG_FORD_CANFD = 2
 
-  def __init__(self, serial: str | None = None, claim: bool = True, disable_checks: bool = True, can_speed_kbps: int = 500):
-    self._connect_serial = serial
+  def __init__(self, serial: str | None = None, claim: bool = True, disable_checks: bool = True, can_speed_kbps: int = 500, cli: bool = True):
     self._disable_checks = disable_checks
 
     self._handle: BaseHandle
     self._handle_open = False
     self.can_rx_overflow_buffer = b''
     self._can_speed_kbps = can_speed_kbps
 
+    if cli and serial is None:
+        self._connect_serial = self._cli_select_panda()
+    else:
+        self._connect_serial = serial
+
     # connect and set mcu type
     self.connect(claim)
 
+  def _cli_select_panda(self):
+    dfu_pandas = PandaDFU.list()
+    if len(dfu_pandas) > 0:
+      print("INFO: some attached pandas are in DFU mode.")
+
+    pandas = self.list()
+    if len(pandas) == 0:
+      print("INFO: panda not available")
+      return None
+    if len(pandas) == 1:
+      print(f"INFO: connecting to panda {pandas[0]}")
+      time.sleep(1)
+      return pandas[0]
+    while True:
+      print("Multiple pandas available:")
+      pandas.sort()
+      for idx, serial in enumerate(pandas):
+        print(f"{[idx]}: {serial}")
+      try:
+        choice = int(input("Choose serial [0]:") or "0")
+        return pandas[choice]
+      except (ValueError, IndexError):
+        print("Enter a valid index.")
+
   def __enter__(self):
     return self
 
@@ -391,6 +419,12 @@ def usb_connect(cls, serial, claim=True, no_error=False):
 
     return context, usb_handle, usb_serial, bootstub, bcd
 
+  def is_connected_spi(self):
+    return isinstance(self._handle, PandaSpiHandle)
+
+  def is_connected_usb(self):
+    return isinstance(self._handle, PandaUsbHandle)
+
   @classmethod
   def list(cls):
     ret = cls.usb_list()

tests/libpanda/SConscript

@@ -21,6 +21,18 @@ env = Environment(
 if system == "Darwin":
   env.PrependENVPath('PATH', '/opt/homebrew/bin')
 
+if GetOption('mutation'):
+  env['CC'] = 'clang-17'
+  flags = [
+    '-fprofile-instr-generate',
+    '-fcoverage-mapping',
+    '-fpass-plugin=/usr/lib/mull-ir-frontend-17',
+    '-g',
+    '-grecord-command-line',
+  ]
+  env['CFLAGS'] += flags
+  env['LINKFLAGS'] += flags
+
 if GetOption('ubsan'):
   flags = [
     "-fsanitize=undefined",

tests/safety/install_mull.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+if ! command -v "mull-runner-17" > /dev/null 2>&1; then
+  sudo apt-get update && sudo apt-get install -y curl clang-17
+  curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
+  sudo apt-get update && sudo apt-get install -y mull-17
+fi

tests/safety/mutation.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+$DIR/install_mull.sh
+
+GIT_REF="${GIT_REF:-origin/master}"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+echo -e "mutators:\n  - cxx_all" > $GIT_ROOT/mull.yml
+scons --mutation -j$(nproc) -D
+echo -e "timeout: 10000\ngitDiffRef: $GIT_REF\ngitProjectRoot: $GIT_ROOT" >> $GIT_ROOT/mull.yml
+
+SAFETY_MODELS=$(find * | grep "^test_.*\.py")
+for safety_model in ${SAFETY_MODELS[@]}; do
+  echo ""
+  echo ""
+  echo -e "Testing mutations on : $safety_model"
+  mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ../libpanda/libpanda.so -test-program=./$safety_model
+done