From b13bdd8c6fca103bdd46b39cc3101968f07f91e0 Mon Sep 17 00:00:00 2001 From: Ivar Date: Thu, 5 Dec 2024 18:16:59 +0000 Subject: [PATCH 01/19] basic fuzzing --- DEVELOPMENT.md | 8 +++ run_functional_tests.sh | 33 +++++----- test/fuzz_api_submissions.py | 117 +++++++++++++++++++++++++++++++++++ test/requirements.txt | 1 + 4 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 test/fuzz_api_submissions.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 304ec160..66e3a26b 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -176,6 +176,14 @@ PLATFORM=virtual CMAKE_BUILD_TYPE=Debug BUILD_CCF_FROM_SOURCE=ON ./build.sh PLATFORM=virtual ./run_functional_tests.sh ``` +### Fuzzing + +Run HTTP API fuzzing tests: + +```sh + +``` + ## AMD SEV-SNP platform To use [AMD SEV-SNP](https://microsoft.github.io/CCF/main/operations/platforms/snp.html) as a platform, it is required to pass additional configuration values required by CCF for the attestation on AMD SEV-SNP hardware. These values may differ depending on which SNP platform you are using (e.g., Confidential Containers on ACI, Confidential Containers on AKS). diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 38158fdf..3f51cca6 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -6,9 +6,8 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} - -# Variable to enable performance tests ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} +RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -44,9 +43,11 @@ if [ "$DOCKER" = "1" ]; then # Turn off pytest output capture to allow test logs to be interleaved with # the docker logs. TEST_ARGS="-s" + FUZZ_ARGS="" else SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" + FUZZ_ARGS="--start-cchost --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution" fi echo "Setting up python virtual environment." @@ -58,17 +59,21 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt -# Enable performance tests if the variable is set -if [ -n "$ENABLE_PERF_TESTS" ]; then - TEST_ARGS="$TEST_ARGS --enable-perf" - echo "Performance tests enabled" +if [ -n "$RUN_FUZZ_TESTS" ]; then + echo "Running fuzz tests..." + python -m test.fuzz_api_submissions $FUZZ_ARGS fi -echo "Running functional tests..." -if [ -n "$ELEVATE_PRIVILEGES" ]; then - sudo -E --preserve-env=PATH \ - capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ - -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" -else - pytest ./test -v -rA $TEST_ARGS "$@" -fi +# if [ -n "$ENABLE_PERF_TESTS" ]; then +# TEST_ARGS="$TEST_ARGS --enable-perf" +# echo "Performance tests enabled" +# fi + +# echo "Running functional tests..." +# if [ -n "$ELEVATE_PRIVILEGES" ]; then +# sudo -E --preserve-env=PATH \ +# capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ +# -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" +# else +# pytest ./test -v -rA $TEST_ARGS "$@" +# fi diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py new file mode 100644 index 00000000..e5a1fc64 --- /dev/null +++ b/test/fuzz_api_submissions.py @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import argparse +import boofuzz +import ssl +from .infra.fixtures import ManagedCCHostFixtures +from .infra.cchost import get_default_cchost_path, get_enclave_path +from pathlib import Path +from loguru import logger as LOG + + +do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) +do_not_verify_tls.check_hostname = False +do_not_verify_tls.verify_mode = ssl.CERT_NONE + +def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): + try: + response = target.recv(10000) + except: + fuzz_data_logger.log_fail("Unable to connect. Target is down.") + return False + + # check response contains a substring foobar + if b"HTTP/1.1 400 BAD_REQUEST" not in response: + fuzz_data_logger.log_fail("Response does not contain 'HTTP/1.1 400 BAD_REQUEST'") + fuzz_data_logger.log_fail("Response: {}".format(response)) + return False + + return True + + +def test_fuzz_api_submissions_random_payload(): + # Create a session and a target to fuzz + session = boofuzz.Session( + target=boofuzz.Target( + connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + ), + post_test_case_callbacks=[response_must_be_400], + receive_data_after_each_request = False, + check_data_received_each_request = False, + receive_data_after_fuzz = False, + ignore_connection_issues_when_sending_fuzz_data = False, + ignore_connection_ssl_errors= True, + reuse_target_connection= False, + sleep_time = 0.01, + ) + + # Create a request variable with fuzzable fields + boofuzz.s_initialize(name="SubmitAny") + with boofuzz.s_block("Request-Line"): + boofuzz.s_static("POST /entries HTTP/1.1\r\n") + boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") + boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") + boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") + boofuzz.s_static("Content-Type: application/cose\r\n") + + # Add claculated content length + boofuzz.s_static("Content-Length: ", name="Content-Length-Header") + boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_static("\r\n", "Content-Length-CRLF") + + boofuzz.s_static("\r\n", "Request-CRLF") + + # Add a fuzzable payload + with boofuzz.s_block("Body-Content"): + boofuzz.s_delim(b'\xD2', name="COSE tag") + boofuzz.s_delim(b'\x84', name="CBOR array tag") + boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB + + # Add a defined request to the session + session.connect(boofuzz.s_get("SubmitAny")) + + # Execute the fuzzing session with all of the session request tests + session.fuzz(max_depth=1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--start-cchost", + action="store_true", + help="Start a cchost process managed by the test framework", + ) + parser.add_argument( + "--enclave-package", + default="/tmp/scitt/lib/libscitt", + help="The enclave package to load.", + ) + parser.add_argument( + "--constitution", + type=Path, + default="/tmp/scitt/share/scitt/constitution", + help="Path to the directory containing the constitution.", + ) + args = parser.parse_args() + + if args.start_cchost: + platform = 'virtual' + enclave_package = args.enclave_package + binary = get_default_cchost_path(platform) + constitution = args.constitution + enclave_file = get_enclave_path(platform, enclave_package) + mccf = ManagedCCHostFixtures( + binary, + platform, + enclave_file, + constitution, + False, + None, + ) + cchost = mccf.start_cchost(None) + + test_fuzz_api_submissions_random_payload() + + \ No newline at end of file diff --git a/test/requirements.txt b/test/requirements.txt index 7725c1d7..e67779e8 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,3 +1,4 @@ +boofuzz locust httpx pytest From b7ce3dddb51d514d880dd925935fa6d72393f886 Mon Sep 17 00:00:00 2001 From: Ivar Date: Tue, 10 Dec 2024 17:06:27 +0000 Subject: [PATCH 02/19] adds file based fuzz test and a job in GH --- .github/workflows/long-test.yml | 30 +++++++++ .gitignore | 1 + DEVELOPMENT.md | 10 ++- run_functional_tests.sh | 31 ++++----- run_fuzz_tests.sh | 64 +++++++++++++++++++ test/fuzz_api_submissions.py | 107 ++++++++++++++++++-------------- 6 files changed, 177 insertions(+), 66 deletions(-) create mode 100755 run_fuzz_tests.sh diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index da256140..885c67e9 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -53,3 +53,33 @@ jobs: /tmp/pytest-of-root/*current/*current/*.{out,err} /tmp/pytest-of-root/*current/*current/config.json if-no-files-found: warn + fuzz-api: + name: fuzz + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} + runs-on: ubuntu-20.04 + container: + image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + env: + PLATFORM: virtual + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + ./build.sh + + - name: Fuzz tests + run: | + set +x + ./run_fuzz_tests.sh + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: boofuzz-results + path: | + boofuzz-results/*.{db} + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 42369002..a56af619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build/ tmp/ out/ venv/ +boofuzz-results/ .venv_ccf_sandbox/ workspace/ *.egg-info/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 66e3a26b..d72b922d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -178,10 +178,18 @@ PLATFORM=virtual ./run_functional_tests.sh ### Fuzzing -Run HTTP API fuzzing tests: +Run HTTP API fuzzing tests after building the application: + +**Using Docker** ```sh +DOCKER=1 ./run_fuzz_tests.sh +``` +**Using your host environment** + +```sh +./run_fuzz_tests.sh ``` ## AMD SEV-SNP platform diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 3f51cca6..56dafa93 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -8,6 +8,7 @@ DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} +SCITT_DIR=${SCITT_DIR:-/tmp/scitt} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -43,11 +44,8 @@ if [ "$DOCKER" = "1" ]; then # Turn off pytest output capture to allow test logs to be interleaved with # the docker logs. TEST_ARGS="-s" - FUZZ_ARGS="" else - SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" - FUZZ_ARGS="--start-cchost --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution" fi echo "Setting up python virtual environment." @@ -59,21 +57,16 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt -if [ -n "$RUN_FUZZ_TESTS" ]; then - echo "Running fuzz tests..." - python -m test.fuzz_api_submissions $FUZZ_ARGS +if [ -n "$ENABLE_PERF_TESTS" ]; then + TEST_ARGS="$TEST_ARGS --enable-perf" + echo "Performance tests enabled" fi -# if [ -n "$ENABLE_PERF_TESTS" ]; then -# TEST_ARGS="$TEST_ARGS --enable-perf" -# echo "Performance tests enabled" -# fi - -# echo "Running functional tests..." -# if [ -n "$ELEVATE_PRIVILEGES" ]; then -# sudo -E --preserve-env=PATH \ -# capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ -# -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" -# else -# pytest ./test -v -rA $TEST_ARGS "$@" -# fi +echo "Running functional tests..." +if [ -n "$ELEVATE_PRIVILEGES" ]; then + sudo -E --preserve-env=PATH \ + capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ + -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" +else + pytest ./test -v -rA $TEST_ARGS "$@" +fi diff --git a/run_fuzz_tests.sh b/run_fuzz_tests.sh new file mode 100755 index 00000000..96dc3244 --- /dev/null +++ b/run_fuzz_tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -ex + +DOCKER=${DOCKER:-0} +PLATFORM=virtual + +wait_for_service() { + url=$1 + timeout=120 + while ! curl -s -f -k "$url" > /dev/null; do + echo "Waiting for service to be ready..." + sleep 1 + timeout=$((timeout - 1)) + if [ $timeout -eq 0 ]; then + echo "Service failed to become ready, exiting" + exit 1 + fi + done +} + +echo "Setting up python virtual environment." +if [ ! -f "venv/bin/activate" ]; then + python3.8 -m venv "venv" +fi +source venv/bin/activate +pip install --disable-pip-version-check -q -e ./pyscitt +pip install --disable-pip-version-check -q wheel +pip install --disable-pip-version-check -q -r test/requirements.txt + +echo "Running fuzz tests..." +export CCF_HOST=${CCF_HOST:-"localhost"} +export CCF_PORT=${CCF_PORT:-8000} +export CCF_URL="https://${CCF_HOST}:${CCF_PORT}" + +if [ "$DOCKER" = "1" ]; then + echo "Will use a running docker instance for testing..." + + PLATFORM=$PLATFORM ./docker/run-dev.sh & + CCF_NETWORK_PID=$! + trap "kill $CCF_NETWORK_PID" EXIT + + wait_for_service "$CCF_URL/parameters" +else + echo "Will use a built SCITT binary for testing..." + + PLATFORM=$PLATFORM ./start.sh & + CCF_PROCESS_PID=$! + trap "kill $CCF_PROCESS_PID" EXIT + + export CCF_URL="https://localhost:8000" + wait_for_service "$CCF_URL/node/network" + + scitt governance local_development \ + --url "$CCF_URL" \ + --member-key workspace/member0_privk.pem \ + --member-cert workspace/member0_cert.pem; + + wait_for_service "$CCF_URL/parameters" +fi + +python -m test.fuzz_api_submissions diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index e5a1fc64..371790e3 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -6,8 +6,8 @@ import ssl from .infra.fixtures import ManagedCCHostFixtures from .infra.cchost import get_default_cchost_path, get_enclave_path +import os from pathlib import Path -from loguru import logger as LOG do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) @@ -31,7 +31,9 @@ def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): def test_fuzz_api_submissions_random_payload(): - # Create a session and a target to fuzz + """ + Generate random payloads and try to register them, each call should return 400 error + """ session = boofuzz.Session( target=boofuzz.Target( connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) @@ -41,9 +43,10 @@ def test_fuzz_api_submissions_random_payload(): check_data_received_each_request = False, receive_data_after_fuzz = False, ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors= True, - reuse_target_connection= False, - sleep_time = 0.01, + ignore_connection_ssl_errors = True, + reuse_target_connection = False, + sleep_time = 0.002, + web_port = None, ) # Create a request variable with fuzzable fields @@ -69,49 +72,61 @@ def test_fuzz_api_submissions_random_payload(): boofuzz.s_delim(b'\x84', name="CBOR array tag") boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB - # Add a defined request to the session - session.connect(boofuzz.s_get("SubmitAny")) - - # Execute the fuzzing session with all of the session request tests - session.fuzz(max_depth=1) + test_request = boofuzz.s_get("SubmitAny") + print("Number of mutations", test_request.num_mutations()) + session.connect(test_request) + session.fuzz(max_depth=2) -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--start-cchost", - action="store_true", - help="Start a cchost process managed by the test framework", - ) - parser.add_argument( - "--enclave-package", - default="/tmp/scitt/lib/libscitt", - help="The enclave package to load.", - ) - parser.add_argument( - "--constitution", - type=Path, - default="/tmp/scitt/share/scitt/constitution", - help="Path to the directory containing the constitution.", +def test_fuzz_api_submissions_cose_payload(): + """ + Randomise parts of the cose envelope and do a successfull submission + """ + current_dir = os.path.dirname(__file__) + + session = boofuzz.Session( + target=boofuzz.Target( + connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + ), + post_test_case_callbacks=[response_must_be_400], + receive_data_after_each_request = False, + check_data_received_each_request = False, + receive_data_after_fuzz = False, + ignore_connection_issues_when_sending_fuzz_data = False, + ignore_connection_ssl_errors = True, + reuse_target_connection = False, + sleep_time = 0.002, + web_port = None, ) - args = parser.parse_args() - - if args.start_cchost: - platform = 'virtual' - enclave_package = args.enclave_package - binary = get_default_cchost_path(platform) - constitution = args.constitution - enclave_file = get_enclave_path(platform, enclave_package) - mccf = ManagedCCHostFixtures( - binary, - platform, - enclave_file, - constitution, - False, - None, - ) - cchost = mccf.start_cchost(None) - - test_fuzz_api_submissions_random_payload() + # Create a request variable with fuzzable fields + boofuzz.s_initialize(name="SubmitCose") + with boofuzz.s_block("Request-Line"): + boofuzz.s_static("POST /entries HTTP/1.1\r\n") + boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") + boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") + boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") + boofuzz.s_static("Content-Type: application/cose\r\n") + # Add claculated content length + boofuzz.s_static("Content-Length: ", name="Content-Length-Header") + boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_static("\r\n", "Content-Length-CRLF") + boofuzz.s_static("\r\n", "Request-CRLF") + + filepath = os.path.join(current_dir, 'payloads/cts-hashv-cwtclaims-b64url.cose') + print('seeding test cose file for fuzzing', filepath) + # Add a fuzzable payload + with boofuzz.s_block("Body-Content"): + boofuzz.s_from_file(filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1<<20) + + test_request = boofuzz.s_get("SubmitCose") + print("Number of mutations", test_request.num_mutations()) + session.connect(test_request) + session.fuzz(max_depth=2) + + +if __name__ == "__main__": + test_fuzz_api_submissions_random_payload() + test_fuzz_api_submissions_cose_payload() \ No newline at end of file From 292038a2a5d729fdf33acf4630e6bc18f74cdebe Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 11:41:41 +0000 Subject: [PATCH 03/19] revert func test script to prior state --- run_functional_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 56dafa93..430fd3bb 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -7,8 +7,6 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} -RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} -SCITT_DIR=${SCITT_DIR:-/tmp/scitt} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -45,6 +43,7 @@ if [ "$DOCKER" = "1" ]; then # the docker logs. TEST_ARGS="-s" else + SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" fi @@ -57,6 +56,7 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt +# Enable performance tests if the variable is set if [ -n "$ENABLE_PERF_TESTS" ]; then TEST_ARGS="$TEST_ARGS --enable-perf" echo "Performance tests enabled" From 2558bca265fbfc7f641b97ab12bb2e98f12c791b Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 11:42:24 +0000 Subject: [PATCH 04/19] revert func test script --- run_functional_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 430fd3bb..38158fdf 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -6,6 +6,8 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} + +# Variable to enable performance tests ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} # SNP attestation config From ec0505e35065816160252fbda2790cb98676177a Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 12:03:54 +0000 Subject: [PATCH 05/19] python formatting --- test/fuzz_api_submissions.py | 110 +++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index 371790e3..dca207c3 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -1,32 +1,31 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. -import argparse -import boofuzz -import ssl -from .infra.fixtures import ManagedCCHostFixtures -from .infra.cchost import get_default_cchost_path, get_enclave_path import os -from pathlib import Path +import ssl +import boofuzz # type: ignore do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) do_not_verify_tls.check_hostname = False do_not_verify_tls.verify_mode = ssl.CERT_NONE + def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): try: response = target.recv(10000) except: fuzz_data_logger.log_fail("Unable to connect. Target is down.") return False - + # check response contains a substring foobar if b"HTTP/1.1 400 BAD_REQUEST" not in response: - fuzz_data_logger.log_fail("Response does not contain 'HTTP/1.1 400 BAD_REQUEST'") + fuzz_data_logger.log_fail( + "Response does not contain 'HTTP/1.1 400 BAD_REQUEST'" + ) fuzz_data_logger.log_fail("Response: {}".format(response)) return False - + return True @@ -36,41 +35,54 @@ def test_fuzz_api_submissions_random_payload(): """ session = boofuzz.Session( target=boofuzz.Target( - connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + connection=boofuzz.SSLSocketConnection( + host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls + ) ), post_test_case_callbacks=[response_must_be_400], - receive_data_after_each_request = False, - check_data_received_each_request = False, - receive_data_after_fuzz = False, - ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors = True, - reuse_target_connection = False, - sleep_time = 0.002, - web_port = None, + receive_data_after_each_request=False, + check_data_received_each_request=False, + receive_data_after_fuzz=False, + ignore_connection_issues_when_sending_fuzz_data=False, + ignore_connection_ssl_errors=True, + reuse_target_connection=False, + sleep_time=0.002, + web_port=None, ) # Create a request variable with fuzzable fields boofuzz.s_initialize(name="SubmitAny") with boofuzz.s_block("Request-Line"): boofuzz.s_static("POST /entries HTTP/1.1\r\n") - boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") - boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static( + "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n" + ) + boofuzz.s_static( + "Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n" + ) boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") boofuzz.s_static("Content-Type: application/cose\r\n") # Add claculated content length boofuzz.s_static("Content-Length: ", name="Content-Length-Header") - boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_size( + "Body-Content", + output_format="ascii", + name="Content-Length-Value", + fuzzable=False, + ) boofuzz.s_static("\r\n", "Content-Length-CRLF") - + boofuzz.s_static("\r\n", "Request-CRLF") # Add a fuzzable payload with boofuzz.s_block("Body-Content"): - boofuzz.s_delim(b'\xD2', name="COSE tag") - boofuzz.s_delim(b'\x84', name="CBOR array tag") - boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB + boofuzz.s_delim(b"\xD2", name="COSE tag") + boofuzz.s_delim(b"\x84", name="CBOR array tag") + boofuzz.s_string( + "Body content ...", name="Body-Content-Value", max_len=(1 << 20 - 2) + ) # 1MB test_request = boofuzz.s_get("SubmitAny") print("Number of mutations", test_request.num_mutations()) @@ -86,39 +98,52 @@ def test_fuzz_api_submissions_cose_payload(): session = boofuzz.Session( target=boofuzz.Target( - connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + connection=boofuzz.SSLSocketConnection( + host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls + ) ), post_test_case_callbacks=[response_must_be_400], - receive_data_after_each_request = False, - check_data_received_each_request = False, - receive_data_after_fuzz = False, - ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors = True, - reuse_target_connection = False, - sleep_time = 0.002, - web_port = None, + receive_data_after_each_request=False, + check_data_received_each_request=False, + receive_data_after_fuzz=False, + ignore_connection_issues_when_sending_fuzz_data=False, + ignore_connection_ssl_errors=True, + reuse_target_connection=False, + sleep_time=0.002, + web_port=None, ) # Create a request variable with fuzzable fields boofuzz.s_initialize(name="SubmitCose") with boofuzz.s_block("Request-Line"): boofuzz.s_static("POST /entries HTTP/1.1\r\n") - boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") - boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static( + "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n" + ) + boofuzz.s_static( + "Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n" + ) boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") boofuzz.s_static("Content-Type: application/cose\r\n") # Add claculated content length boofuzz.s_static("Content-Length: ", name="Content-Length-Header") - boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_size( + "Body-Content", + output_format="ascii", + name="Content-Length-Value", + fuzzable=False, + ) boofuzz.s_static("\r\n", "Content-Length-CRLF") boofuzz.s_static("\r\n", "Request-CRLF") - filepath = os.path.join(current_dir, 'payloads/cts-hashv-cwtclaims-b64url.cose') - print('seeding test cose file for fuzzing', filepath) + filepath = os.path.join(current_dir, "payloads/cts-hashv-cwtclaims-b64url.cose") + print("seeding test cose file for fuzzing", filepath) # Add a fuzzable payload with boofuzz.s_block("Body-Content"): - boofuzz.s_from_file(filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1<<20) + boofuzz.s_from_file( + filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1 << 20 + ) test_request = boofuzz.s_get("SubmitCose") print("Number of mutations", test_request.num_mutations()) @@ -129,4 +154,3 @@ def test_fuzz_api_submissions_cose_payload(): if __name__ == "__main__": test_fuzz_api_submissions_random_payload() test_fuzz_api_submissions_cose_payload() - \ No newline at end of file From b448f41e198ba4655254d16a5dbb89a14c66fff0 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 12:06:22 +0000 Subject: [PATCH 06/19] change container image for the build to pass --- .github/workflows/long-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 885c67e9..1dba5a2a 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -16,8 +16,7 @@ jobs: name: ASAN if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: - image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + container: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 env: # Fast unwinder only gives us partial stack traces in LeakSanitzer # Alloc/dealloc mismatch has been disabled in CCF: https://github.com/microsoft/CCF/pull/5157 @@ -57,8 +56,7 @@ jobs: name: fuzz if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: - image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev7 env: PLATFORM: virtual steps: From edd6ff48614634e65b3b639a046eb06181868643 Mon Sep 17 00:00:00 2001 From: Ivar Date: Thu, 5 Dec 2024 18:16:59 +0000 Subject: [PATCH 07/19] basic fuzzing --- DEVELOPMENT.md | 8 +++ run_functional_tests.sh | 33 +++++----- test/fuzz_api_submissions.py | 117 +++++++++++++++++++++++++++++++++++ test/requirements.txt | 1 + 4 files changed, 145 insertions(+), 14 deletions(-) create mode 100644 test/fuzz_api_submissions.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a53b6939..fd513565 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -176,6 +176,14 @@ PLATFORM=virtual CMAKE_BUILD_TYPE=Debug BUILD_CCF_FROM_SOURCE=ON ./build.sh PLATFORM=virtual ./run_functional_tests.sh ``` +### Fuzzing + +Run HTTP API fuzzing tests: + +```sh + +``` + ## AMD SEV-SNP platform To use [AMD SEV-SNP](https://microsoft.github.io/CCF/main/operations/platforms/snp.html) as a platform, it is required to pass additional configuration values required by CCF for the attestation on AMD SEV-SNP hardware. These values may differ depending on which SNP platform you are using (e.g., Confidential Containers on ACI, Confidential Containers on AKS). diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 38158fdf..3f51cca6 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -6,9 +6,8 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} - -# Variable to enable performance tests ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} +RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -44,9 +43,11 @@ if [ "$DOCKER" = "1" ]; then # Turn off pytest output capture to allow test logs to be interleaved with # the docker logs. TEST_ARGS="-s" + FUZZ_ARGS="" else SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" + FUZZ_ARGS="--start-cchost --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution" fi echo "Setting up python virtual environment." @@ -58,17 +59,21 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt -# Enable performance tests if the variable is set -if [ -n "$ENABLE_PERF_TESTS" ]; then - TEST_ARGS="$TEST_ARGS --enable-perf" - echo "Performance tests enabled" +if [ -n "$RUN_FUZZ_TESTS" ]; then + echo "Running fuzz tests..." + python -m test.fuzz_api_submissions $FUZZ_ARGS fi -echo "Running functional tests..." -if [ -n "$ELEVATE_PRIVILEGES" ]; then - sudo -E --preserve-env=PATH \ - capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ - -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" -else - pytest ./test -v -rA $TEST_ARGS "$@" -fi +# if [ -n "$ENABLE_PERF_TESTS" ]; then +# TEST_ARGS="$TEST_ARGS --enable-perf" +# echo "Performance tests enabled" +# fi + +# echo "Running functional tests..." +# if [ -n "$ELEVATE_PRIVILEGES" ]; then +# sudo -E --preserve-env=PATH \ +# capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ +# -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" +# else +# pytest ./test -v -rA $TEST_ARGS "$@" +# fi diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py new file mode 100644 index 00000000..e5a1fc64 --- /dev/null +++ b/test/fuzz_api_submissions.py @@ -0,0 +1,117 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +import argparse +import boofuzz +import ssl +from .infra.fixtures import ManagedCCHostFixtures +from .infra.cchost import get_default_cchost_path, get_enclave_path +from pathlib import Path +from loguru import logger as LOG + + +do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) +do_not_verify_tls.check_hostname = False +do_not_verify_tls.verify_mode = ssl.CERT_NONE + +def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): + try: + response = target.recv(10000) + except: + fuzz_data_logger.log_fail("Unable to connect. Target is down.") + return False + + # check response contains a substring foobar + if b"HTTP/1.1 400 BAD_REQUEST" not in response: + fuzz_data_logger.log_fail("Response does not contain 'HTTP/1.1 400 BAD_REQUEST'") + fuzz_data_logger.log_fail("Response: {}".format(response)) + return False + + return True + + +def test_fuzz_api_submissions_random_payload(): + # Create a session and a target to fuzz + session = boofuzz.Session( + target=boofuzz.Target( + connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + ), + post_test_case_callbacks=[response_must_be_400], + receive_data_after_each_request = False, + check_data_received_each_request = False, + receive_data_after_fuzz = False, + ignore_connection_issues_when_sending_fuzz_data = False, + ignore_connection_ssl_errors= True, + reuse_target_connection= False, + sleep_time = 0.01, + ) + + # Create a request variable with fuzzable fields + boofuzz.s_initialize(name="SubmitAny") + with boofuzz.s_block("Request-Line"): + boofuzz.s_static("POST /entries HTTP/1.1\r\n") + boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") + boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") + boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") + boofuzz.s_static("Content-Type: application/cose\r\n") + + # Add claculated content length + boofuzz.s_static("Content-Length: ", name="Content-Length-Header") + boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_static("\r\n", "Content-Length-CRLF") + + boofuzz.s_static("\r\n", "Request-CRLF") + + # Add a fuzzable payload + with boofuzz.s_block("Body-Content"): + boofuzz.s_delim(b'\xD2', name="COSE tag") + boofuzz.s_delim(b'\x84', name="CBOR array tag") + boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB + + # Add a defined request to the session + session.connect(boofuzz.s_get("SubmitAny")) + + # Execute the fuzzing session with all of the session request tests + session.fuzz(max_depth=1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--start-cchost", + action="store_true", + help="Start a cchost process managed by the test framework", + ) + parser.add_argument( + "--enclave-package", + default="/tmp/scitt/lib/libscitt", + help="The enclave package to load.", + ) + parser.add_argument( + "--constitution", + type=Path, + default="/tmp/scitt/share/scitt/constitution", + help="Path to the directory containing the constitution.", + ) + args = parser.parse_args() + + if args.start_cchost: + platform = 'virtual' + enclave_package = args.enclave_package + binary = get_default_cchost_path(platform) + constitution = args.constitution + enclave_file = get_enclave_path(platform, enclave_package) + mccf = ManagedCCHostFixtures( + binary, + platform, + enclave_file, + constitution, + False, + None, + ) + cchost = mccf.start_cchost(None) + + test_fuzz_api_submissions_random_payload() + + \ No newline at end of file diff --git a/test/requirements.txt b/test/requirements.txt index 8e9abd10..e6c77d03 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,3 +1,4 @@ +boofuzz locust httpx pytest From 3412779060069fc9273495fe6074842d8231ccd4 Mon Sep 17 00:00:00 2001 From: Ivar Date: Tue, 10 Dec 2024 17:06:27 +0000 Subject: [PATCH 08/19] adds file based fuzz test and a job in GH --- .github/workflows/long-test.yml | 30 +++++++++ .gitignore | 1 + DEVELOPMENT.md | 10 ++- run_functional_tests.sh | 31 ++++----- run_fuzz_tests.sh | 64 +++++++++++++++++++ test/fuzz_api_submissions.py | 107 ++++++++++++++++++-------------- 6 files changed, 177 insertions(+), 66 deletions(-) create mode 100755 run_fuzz_tests.sh diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index da256140..885c67e9 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -53,3 +53,33 @@ jobs: /tmp/pytest-of-root/*current/*current/*.{out,err} /tmp/pytest-of-root/*current/*current/config.json if-no-files-found: warn + fuzz-api: + name: fuzz + if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} + runs-on: ubuntu-20.04 + container: + image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + env: + PLATFORM: virtual + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + ./build.sh + + - name: Fuzz tests + run: | + set +x + ./run_fuzz_tests.sh + + - name: "Upload logs" + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: boofuzz-results + path: | + boofuzz-results/*.{db} + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 42369002..a56af619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build/ tmp/ out/ venv/ +boofuzz-results/ .venv_ccf_sandbox/ workspace/ *.egg-info/ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index fd513565..118a76b3 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -178,10 +178,18 @@ PLATFORM=virtual ./run_functional_tests.sh ### Fuzzing -Run HTTP API fuzzing tests: +Run HTTP API fuzzing tests after building the application: + +**Using Docker** ```sh +DOCKER=1 ./run_fuzz_tests.sh +``` +**Using your host environment** + +```sh +./run_fuzz_tests.sh ``` ## AMD SEV-SNP platform diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 3f51cca6..56dafa93 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -8,6 +8,7 @@ DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} +SCITT_DIR=${SCITT_DIR:-/tmp/scitt} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -43,11 +44,8 @@ if [ "$DOCKER" = "1" ]; then # Turn off pytest output capture to allow test logs to be interleaved with # the docker logs. TEST_ARGS="-s" - FUZZ_ARGS="" else - SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" - FUZZ_ARGS="--start-cchost --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution" fi echo "Setting up python virtual environment." @@ -59,21 +57,16 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt -if [ -n "$RUN_FUZZ_TESTS" ]; then - echo "Running fuzz tests..." - python -m test.fuzz_api_submissions $FUZZ_ARGS +if [ -n "$ENABLE_PERF_TESTS" ]; then + TEST_ARGS="$TEST_ARGS --enable-perf" + echo "Performance tests enabled" fi -# if [ -n "$ENABLE_PERF_TESTS" ]; then -# TEST_ARGS="$TEST_ARGS --enable-perf" -# echo "Performance tests enabled" -# fi - -# echo "Running functional tests..." -# if [ -n "$ELEVATE_PRIVILEGES" ]; then -# sudo -E --preserve-env=PATH \ -# capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ -# -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" -# else -# pytest ./test -v -rA $TEST_ARGS "$@" -# fi +echo "Running functional tests..." +if [ -n "$ELEVATE_PRIVILEGES" ]; then + sudo -E --preserve-env=PATH \ + capsh --keep=1 --user=$(id -un) --inh=cap_net_bind_service --addamb=cap_net_bind_service \ + -- -c "pytest ./test -v -rA $TEST_ARGS $(printf "'%s' " "$@")" +else + pytest ./test -v -rA $TEST_ARGS "$@" +fi diff --git a/run_fuzz_tests.sh b/run_fuzz_tests.sh new file mode 100755 index 00000000..96dc3244 --- /dev/null +++ b/run_fuzz_tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -ex + +DOCKER=${DOCKER:-0} +PLATFORM=virtual + +wait_for_service() { + url=$1 + timeout=120 + while ! curl -s -f -k "$url" > /dev/null; do + echo "Waiting for service to be ready..." + sleep 1 + timeout=$((timeout - 1)) + if [ $timeout -eq 0 ]; then + echo "Service failed to become ready, exiting" + exit 1 + fi + done +} + +echo "Setting up python virtual environment." +if [ ! -f "venv/bin/activate" ]; then + python3.8 -m venv "venv" +fi +source venv/bin/activate +pip install --disable-pip-version-check -q -e ./pyscitt +pip install --disable-pip-version-check -q wheel +pip install --disable-pip-version-check -q -r test/requirements.txt + +echo "Running fuzz tests..." +export CCF_HOST=${CCF_HOST:-"localhost"} +export CCF_PORT=${CCF_PORT:-8000} +export CCF_URL="https://${CCF_HOST}:${CCF_PORT}" + +if [ "$DOCKER" = "1" ]; then + echo "Will use a running docker instance for testing..." + + PLATFORM=$PLATFORM ./docker/run-dev.sh & + CCF_NETWORK_PID=$! + trap "kill $CCF_NETWORK_PID" EXIT + + wait_for_service "$CCF_URL/parameters" +else + echo "Will use a built SCITT binary for testing..." + + PLATFORM=$PLATFORM ./start.sh & + CCF_PROCESS_PID=$! + trap "kill $CCF_PROCESS_PID" EXIT + + export CCF_URL="https://localhost:8000" + wait_for_service "$CCF_URL/node/network" + + scitt governance local_development \ + --url "$CCF_URL" \ + --member-key workspace/member0_privk.pem \ + --member-cert workspace/member0_cert.pem; + + wait_for_service "$CCF_URL/parameters" +fi + +python -m test.fuzz_api_submissions diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index e5a1fc64..371790e3 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -6,8 +6,8 @@ import ssl from .infra.fixtures import ManagedCCHostFixtures from .infra.cchost import get_default_cchost_path, get_enclave_path +import os from pathlib import Path -from loguru import logger as LOG do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) @@ -31,7 +31,9 @@ def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): def test_fuzz_api_submissions_random_payload(): - # Create a session and a target to fuzz + """ + Generate random payloads and try to register them, each call should return 400 error + """ session = boofuzz.Session( target=boofuzz.Target( connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) @@ -41,9 +43,10 @@ def test_fuzz_api_submissions_random_payload(): check_data_received_each_request = False, receive_data_after_fuzz = False, ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors= True, - reuse_target_connection= False, - sleep_time = 0.01, + ignore_connection_ssl_errors = True, + reuse_target_connection = False, + sleep_time = 0.002, + web_port = None, ) # Create a request variable with fuzzable fields @@ -69,49 +72,61 @@ def test_fuzz_api_submissions_random_payload(): boofuzz.s_delim(b'\x84', name="CBOR array tag") boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB - # Add a defined request to the session - session.connect(boofuzz.s_get("SubmitAny")) - - # Execute the fuzzing session with all of the session request tests - session.fuzz(max_depth=1) + test_request = boofuzz.s_get("SubmitAny") + print("Number of mutations", test_request.num_mutations()) + session.connect(test_request) + session.fuzz(max_depth=2) -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--start-cchost", - action="store_true", - help="Start a cchost process managed by the test framework", - ) - parser.add_argument( - "--enclave-package", - default="/tmp/scitt/lib/libscitt", - help="The enclave package to load.", - ) - parser.add_argument( - "--constitution", - type=Path, - default="/tmp/scitt/share/scitt/constitution", - help="Path to the directory containing the constitution.", +def test_fuzz_api_submissions_cose_payload(): + """ + Randomise parts of the cose envelope and do a successfull submission + """ + current_dir = os.path.dirname(__file__) + + session = boofuzz.Session( + target=boofuzz.Target( + connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + ), + post_test_case_callbacks=[response_must_be_400], + receive_data_after_each_request = False, + check_data_received_each_request = False, + receive_data_after_fuzz = False, + ignore_connection_issues_when_sending_fuzz_data = False, + ignore_connection_ssl_errors = True, + reuse_target_connection = False, + sleep_time = 0.002, + web_port = None, ) - args = parser.parse_args() - - if args.start_cchost: - platform = 'virtual' - enclave_package = args.enclave_package - binary = get_default_cchost_path(platform) - constitution = args.constitution - enclave_file = get_enclave_path(platform, enclave_package) - mccf = ManagedCCHostFixtures( - binary, - platform, - enclave_file, - constitution, - False, - None, - ) - cchost = mccf.start_cchost(None) - - test_fuzz_api_submissions_random_payload() + # Create a request variable with fuzzable fields + boofuzz.s_initialize(name="SubmitCose") + with boofuzz.s_block("Request-Line"): + boofuzz.s_static("POST /entries HTTP/1.1\r\n") + boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") + boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") + boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") + boofuzz.s_static("Content-Type: application/cose\r\n") + # Add claculated content length + boofuzz.s_static("Content-Length: ", name="Content-Length-Header") + boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_static("\r\n", "Content-Length-CRLF") + boofuzz.s_static("\r\n", "Request-CRLF") + + filepath = os.path.join(current_dir, 'payloads/cts-hashv-cwtclaims-b64url.cose') + print('seeding test cose file for fuzzing', filepath) + # Add a fuzzable payload + with boofuzz.s_block("Body-Content"): + boofuzz.s_from_file(filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1<<20) + + test_request = boofuzz.s_get("SubmitCose") + print("Number of mutations", test_request.num_mutations()) + session.connect(test_request) + session.fuzz(max_depth=2) + + +if __name__ == "__main__": + test_fuzz_api_submissions_random_payload() + test_fuzz_api_submissions_cose_payload() \ No newline at end of file From ef4865514eb085b3b01cbbaf88356ecaa42b7337 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 11:41:41 +0000 Subject: [PATCH 09/19] revert func test script to prior state --- run_functional_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 56dafa93..430fd3bb 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -7,8 +7,6 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} -RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-} -SCITT_DIR=${SCITT_DIR:-/tmp/scitt} # SNP attestation config SNP_ATTESTATION_CONFIG=${SNP_ATTESTATION_CONFIG:-} @@ -45,6 +43,7 @@ if [ "$DOCKER" = "1" ]; then # the docker logs. TEST_ARGS="-s" else + SCITT_DIR=/tmp/scitt TEST_ARGS="--start-cchost --platform=$PLATFORM --enclave-package=$SCITT_DIR/lib/libscitt --constitution=$SCITT_DIR/share/scitt/constitution --snp-attestation-config=$SNP_ATTESTATION_CONFIG" fi @@ -57,6 +56,7 @@ pip install --disable-pip-version-check -q -e ./pyscitt pip install --disable-pip-version-check -q wheel pip install --disable-pip-version-check -q -r test/requirements.txt +# Enable performance tests if the variable is set if [ -n "$ENABLE_PERF_TESTS" ]; then TEST_ARGS="$TEST_ARGS --enable-perf" echo "Performance tests enabled" From 3a890a9aca46ab2e04d226bf58429b59278a5b78 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 11:42:24 +0000 Subject: [PATCH 10/19] revert func test script --- run_functional_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 430fd3bb..38158fdf 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -6,6 +6,8 @@ set -ex DOCKER=${DOCKER:-0} PLATFORM=${PLATFORM:-snp} + +# Variable to enable performance tests ENABLE_PERF_TESTS=${ENABLE_PERF_TESTS:-} # SNP attestation config From 10ed7d61045aca35924821419eedff22034010d2 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 12:03:54 +0000 Subject: [PATCH 11/19] python formatting --- test/fuzz_api_submissions.py | 110 +++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index 371790e3..dca207c3 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -1,32 +1,31 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. -import argparse -import boofuzz -import ssl -from .infra.fixtures import ManagedCCHostFixtures -from .infra.cchost import get_default_cchost_path, get_enclave_path import os -from pathlib import Path +import ssl +import boofuzz # type: ignore do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) do_not_verify_tls.check_hostname = False do_not_verify_tls.verify_mode = ssl.CERT_NONE + def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): try: response = target.recv(10000) except: fuzz_data_logger.log_fail("Unable to connect. Target is down.") return False - + # check response contains a substring foobar if b"HTTP/1.1 400 BAD_REQUEST" not in response: - fuzz_data_logger.log_fail("Response does not contain 'HTTP/1.1 400 BAD_REQUEST'") + fuzz_data_logger.log_fail( + "Response does not contain 'HTTP/1.1 400 BAD_REQUEST'" + ) fuzz_data_logger.log_fail("Response: {}".format(response)) return False - + return True @@ -36,41 +35,54 @@ def test_fuzz_api_submissions_random_payload(): """ session = boofuzz.Session( target=boofuzz.Target( - connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + connection=boofuzz.SSLSocketConnection( + host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls + ) ), post_test_case_callbacks=[response_must_be_400], - receive_data_after_each_request = False, - check_data_received_each_request = False, - receive_data_after_fuzz = False, - ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors = True, - reuse_target_connection = False, - sleep_time = 0.002, - web_port = None, + receive_data_after_each_request=False, + check_data_received_each_request=False, + receive_data_after_fuzz=False, + ignore_connection_issues_when_sending_fuzz_data=False, + ignore_connection_ssl_errors=True, + reuse_target_connection=False, + sleep_time=0.002, + web_port=None, ) # Create a request variable with fuzzable fields boofuzz.s_initialize(name="SubmitAny") with boofuzz.s_block("Request-Line"): boofuzz.s_static("POST /entries HTTP/1.1\r\n") - boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") - boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static( + "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n" + ) + boofuzz.s_static( + "Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n" + ) boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") boofuzz.s_static("Content-Type: application/cose\r\n") # Add claculated content length boofuzz.s_static("Content-Length: ", name="Content-Length-Header") - boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_size( + "Body-Content", + output_format="ascii", + name="Content-Length-Value", + fuzzable=False, + ) boofuzz.s_static("\r\n", "Content-Length-CRLF") - + boofuzz.s_static("\r\n", "Request-CRLF") # Add a fuzzable payload with boofuzz.s_block("Body-Content"): - boofuzz.s_delim(b'\xD2', name="COSE tag") - boofuzz.s_delim(b'\x84', name="CBOR array tag") - boofuzz.s_string("Body content ...", name="Body-Content-Value", max_len=(1<<20 - 2)) # 1MB + boofuzz.s_delim(b"\xD2", name="COSE tag") + boofuzz.s_delim(b"\x84", name="CBOR array tag") + boofuzz.s_string( + "Body content ...", name="Body-Content-Value", max_len=(1 << 20 - 2) + ) # 1MB test_request = boofuzz.s_get("SubmitAny") print("Number of mutations", test_request.num_mutations()) @@ -86,39 +98,52 @@ def test_fuzz_api_submissions_cose_payload(): session = boofuzz.Session( target=boofuzz.Target( - connection=boofuzz.SSLSocketConnection(host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls) + connection=boofuzz.SSLSocketConnection( + host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls + ) ), post_test_case_callbacks=[response_must_be_400], - receive_data_after_each_request = False, - check_data_received_each_request = False, - receive_data_after_fuzz = False, - ignore_connection_issues_when_sending_fuzz_data = False, - ignore_connection_ssl_errors = True, - reuse_target_connection = False, - sleep_time = 0.002, - web_port = None, + receive_data_after_each_request=False, + check_data_received_each_request=False, + receive_data_after_fuzz=False, + ignore_connection_issues_when_sending_fuzz_data=False, + ignore_connection_ssl_errors=True, + reuse_target_connection=False, + sleep_time=0.002, + web_port=None, ) # Create a request variable with fuzzable fields boofuzz.s_initialize(name="SubmitCose") with boofuzz.s_block("Request-Line"): boofuzz.s_static("POST /entries HTTP/1.1\r\n") - boofuzz.s_static("User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n") - boofuzz.s_static("Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n") + boofuzz.s_static( + "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:65.0) Gecko/20100101 Firefox/65.0\r\n" + ) + boofuzz.s_static( + "Accept: text/html,application/json;q=0.9,image/webp,*/*;q=0.8\r\n" + ) boofuzz.s_static("Accept-Language: en-US,en;q=0.5\r\n") boofuzz.s_static("Accept-Encoding: gzip, deflate\r\n") boofuzz.s_static("Content-Type: application/cose\r\n") # Add claculated content length boofuzz.s_static("Content-Length: ", name="Content-Length-Header") - boofuzz.s_size("Body-Content", output_format="ascii", name="Content-Length-Value", fuzzable=False) + boofuzz.s_size( + "Body-Content", + output_format="ascii", + name="Content-Length-Value", + fuzzable=False, + ) boofuzz.s_static("\r\n", "Content-Length-CRLF") boofuzz.s_static("\r\n", "Request-CRLF") - filepath = os.path.join(current_dir, 'payloads/cts-hashv-cwtclaims-b64url.cose') - print('seeding test cose file for fuzzing', filepath) + filepath = os.path.join(current_dir, "payloads/cts-hashv-cwtclaims-b64url.cose") + print("seeding test cose file for fuzzing", filepath) # Add a fuzzable payload with boofuzz.s_block("Body-Content"): - boofuzz.s_from_file(filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1<<20) + boofuzz.s_from_file( + filename=filepath, name="Body-Content-Value", fuzzable=True, max_len=1 << 20 + ) test_request = boofuzz.s_get("SubmitCose") print("Number of mutations", test_request.num_mutations()) @@ -129,4 +154,3 @@ def test_fuzz_api_submissions_cose_payload(): if __name__ == "__main__": test_fuzz_api_submissions_random_payload() test_fuzz_api_submissions_cose_payload() - \ No newline at end of file From af56c091da0f1719254aad44461f59d06115dfcc Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 12:06:22 +0000 Subject: [PATCH 12/19] change container image for the build to pass --- .github/workflows/long-test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 885c67e9..1dba5a2a 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -16,8 +16,7 @@ jobs: name: ASAN if: ${{ contains(github.event.pull_request.labels.*.name, 'run-long-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: - image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + container: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 env: # Fast unwinder only gives us partial stack traces in LeakSanitzer # Alloc/dealloc mismatch has been disabled in CCF: https://github.com/microsoft/CCF/pull/5157 @@ -57,8 +56,7 @@ jobs: name: fuzz if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: - image: ghcr.io/microsoft/ccf/ci/default:build-08-10-2024 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev7 env: PLATFORM: virtual steps: From badf7026dfdb4b4847b199ce7eb75e410e3a4921 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 12:11:07 +0000 Subject: [PATCH 13/19] change container to 6.0.0-dev8 --- .github/workflows/long-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 1dba5a2a..94a2a987 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -56,7 +56,7 @@ jobs: name: fuzz if: ${{ contains(github.event.pull_request.labels.*.name, 'run-fuzz-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }} runs-on: ubuntu-20.04 - container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev7 + container: ghcr.io/microsoft/ccf/app/dev/virtual:ccf-6.0.0-dev8 env: PLATFORM: virtual steps: From 7d5f07ece59d86d2f0cd3b610838f1ab0cbebdbe Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 13:50:40 +0000 Subject: [PATCH 14/19] adds timeout to fuzz tests --- test/fuzz_api_submissions.py | 48 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index dca207c3..3d4894e8 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -3,30 +3,46 @@ import os import ssl +import time import boofuzz # type: ignore do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) do_not_verify_tls.check_hostname = False do_not_verify_tls.verify_mode = ssl.CERT_NONE +test_time_threshold_sec = 300 -def response_must_be_400(target, fuzz_data_logger, session, *args, **kwargs): - try: - response = target.recv(10000) - except: - fuzz_data_logger.log_fail("Unable to connect. Target is down.") - return False +def response_must_be_400(): - # check response contains a substring foobar - if b"HTTP/1.1 400 BAD_REQUEST" not in response: - fuzz_data_logger.log_fail( - "Response does not contain 'HTTP/1.1 400 BAD_REQUEST'" - ) - fuzz_data_logger.log_fail("Response: {}".format(response)) - return False + # save current time in python + start_time_sec = time.time() + + def checker(target, fuzz_data_logger, session, *args, **kwargs): + try: + response = target.recv(10000) + except: + fuzz_data_logger.log_fail("Unable to connect. Target is down.") + return False + + result = True + # check response contains a substring foobar + if b"HTTP/1.1 400 BAD_REQUEST" not in response: + fuzz_data_logger.log_fail( + "Response does not contain 'HTTP/1.1 400 BAD_REQUEST'" + ) + fuzz_data_logger.log_fail("Response: {}".format(response)) + result = False + + if time.time() - start_time_sec > test_time_threshold_sec: + fuzz_data_logger.log_info("Timeout reached") + session._index_end = ( + 0 # stop fuzzing https://github.com/jtpereyda/boofuzz/discussions/600 + ) + + return result - return True + return checker def test_fuzz_api_submissions_random_payload(): @@ -39,7 +55,7 @@ def test_fuzz_api_submissions_random_payload(): host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls ) ), - post_test_case_callbacks=[response_must_be_400], + post_test_case_callbacks=[response_must_be_400()], receive_data_after_each_request=False, check_data_received_each_request=False, receive_data_after_fuzz=False, @@ -102,7 +118,7 @@ def test_fuzz_api_submissions_cose_payload(): host="127.0.0.1", port=8000, sslcontext=do_not_verify_tls ) ), - post_test_case_callbacks=[response_must_be_400], + post_test_case_callbacks=[response_must_be_400()], receive_data_after_each_request=False, check_data_received_each_request=False, receive_data_after_fuzz=False, From 63ac80a3c096cfb7b1b7f4446da2783fdd753255 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 15:10:33 +0000 Subject: [PATCH 15/19] add more details to the finished test output and reduce test exec time --- test/fuzz_api_submissions.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index 3d4894e8..c0cd0680 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -10,7 +10,7 @@ do_not_verify_tls = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) do_not_verify_tls.check_hostname = False do_not_verify_tls.verify_mode = ssl.CERT_NONE -test_time_threshold_sec = 300 +test_time_threshold_sec = 180 def response_must_be_400(): @@ -19,20 +19,20 @@ def response_must_be_400(): start_time_sec = time.time() def checker(target, fuzz_data_logger, session, *args, **kwargs): + is_success = True try: response = target.recv(10000) except: fuzz_data_logger.log_fail("Unable to connect. Target is down.") - return False + is_success = False - result = True # check response contains a substring foobar - if b"HTTP/1.1 400 BAD_REQUEST" not in response: + if is_success and b"HTTP/1.1 400 BAD_REQUEST" not in response: fuzz_data_logger.log_fail( "Response does not contain 'HTTP/1.1 400 BAD_REQUEST'" ) fuzz_data_logger.log_fail("Response: {}".format(response)) - result = False + is_success = False if time.time() - start_time_sec > test_time_threshold_sec: fuzz_data_logger.log_info("Timeout reached") @@ -40,7 +40,7 @@ def checker(target, fuzz_data_logger, session, *args, **kwargs): 0 # stop fuzzing https://github.com/jtpereyda/boofuzz/discussions/600 ) - return result + return is_success return checker @@ -101,9 +101,11 @@ def test_fuzz_api_submissions_random_payload(): ) # 1MB test_request = boofuzz.s_get("SubmitAny") - print("Number of mutations", test_request.num_mutations()) + session._fuzz_data_logger.log_info("Number of mutations: {}".format(test_request.num_mutations())) session.connect(test_request) session.fuzz(max_depth=2) + session._fuzz_data_logger.log_info("Number of tests executed: {}".format(session.num_cases_actually_fuzzed)) + session._fuzz_data_logger.log_info("Execution speed: {}".format(session.exec_speed)) def test_fuzz_api_submissions_cose_payload(): @@ -154,7 +156,7 @@ def test_fuzz_api_submissions_cose_payload(): boofuzz.s_static("\r\n", "Request-CRLF") filepath = os.path.join(current_dir, "payloads/cts-hashv-cwtclaims-b64url.cose") - print("seeding test cose file for fuzzing", filepath) + session._fuzz_data_logger.log_info("Seeding test cose file for fuzzing: {}".format(filepath)) # Add a fuzzable payload with boofuzz.s_block("Body-Content"): boofuzz.s_from_file( @@ -162,9 +164,11 @@ def test_fuzz_api_submissions_cose_payload(): ) test_request = boofuzz.s_get("SubmitCose") - print("Number of mutations", test_request.num_mutations()) + session._fuzz_data_logger.log_info("Number of mutations: {}".format(test_request.num_mutations())) session.connect(test_request) session.fuzz(max_depth=2) + session._fuzz_data_logger.log_info("Number of tests executed: {}".format(session.num_cases_actually_fuzzed)) + session._fuzz_data_logger.log_info("Execution speed: {}".format(session.exec_speed)) if __name__ == "__main__": From c9e5b77a1bd1a26e5a28dc1e59a12106cf2920ac Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 15:14:48 +0000 Subject: [PATCH 16/19] bump cryptography dependency due to CCF change --- pyscitt/setup.py | 4 ++-- test/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyscitt/setup.py b/pyscitt/setup.py index f2b7a373..1f53836c 100644 --- a/pyscitt/setup.py +++ b/pyscitt/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup PACKAGE_NAME = "pyscitt" -PACKAGE_VERSION = "0.7.0" +PACKAGE_VERSION = "0.7.1" path_here = path.abspath(path.dirname(__file__)) @@ -26,7 +26,7 @@ python_requires=">=3.8", install_requires=[ "ccf==6.0.0-dev8", - "cryptography==43.*", # needs to match ccf + "cryptography==44.*", # needs to match ccf "httpx", "cbor2==5.4.*", "pycose==1.1.0", diff --git a/test/requirements.txt b/test/requirements.txt index e6c77d03..ba2b45c9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -5,4 +5,4 @@ pytest loguru aiotools ccf==6.0.0-dev8 -cryptography==43.* +cryptography==44.* From f307afba22b2ae32ecea816bf296e6c5313507dc Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 15:18:17 +0000 Subject: [PATCH 17/19] reformat --- test/fuzz_api_submissions.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index c0cd0680..c159a587 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -101,10 +101,14 @@ def test_fuzz_api_submissions_random_payload(): ) # 1MB test_request = boofuzz.s_get("SubmitAny") - session._fuzz_data_logger.log_info("Number of mutations: {}".format(test_request.num_mutations())) + session._fuzz_data_logger.log_info( + "Number of mutations: {}".format(test_request.num_mutations()) + ) session.connect(test_request) session.fuzz(max_depth=2) - session._fuzz_data_logger.log_info("Number of tests executed: {}".format(session.num_cases_actually_fuzzed)) + session._fuzz_data_logger.log_info( + "Number of tests executed: {}".format(session.num_cases_actually_fuzzed) + ) session._fuzz_data_logger.log_info("Execution speed: {}".format(session.exec_speed)) @@ -156,7 +160,9 @@ def test_fuzz_api_submissions_cose_payload(): boofuzz.s_static("\r\n", "Request-CRLF") filepath = os.path.join(current_dir, "payloads/cts-hashv-cwtclaims-b64url.cose") - session._fuzz_data_logger.log_info("Seeding test cose file for fuzzing: {}".format(filepath)) + session._fuzz_data_logger.log_info( + "Seeding test cose file for fuzzing: {}".format(filepath) + ) # Add a fuzzable payload with boofuzz.s_block("Body-Content"): boofuzz.s_from_file( @@ -164,10 +170,14 @@ def test_fuzz_api_submissions_cose_payload(): ) test_request = boofuzz.s_get("SubmitCose") - session._fuzz_data_logger.log_info("Number of mutations: {}".format(test_request.num_mutations())) + session._fuzz_data_logger.log_info( + "Number of mutations: {}".format(test_request.num_mutations()) + ) session.connect(test_request) session.fuzz(max_depth=2) - session._fuzz_data_logger.log_info("Number of tests executed: {}".format(session.num_cases_actually_fuzzed)) + session._fuzz_data_logger.log_info( + "Number of tests executed: {}".format(session.num_cases_actually_fuzzed) + ) session._fuzz_data_logger.log_info("Execution speed: {}".format(session.exec_speed)) From 5daa6591f3aadcecd3f65999ed32339e4d7fa062 Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 16:12:03 +0000 Subject: [PATCH 18/19] improve logging in GH actions --- .github/workflows/long-test.yml | 6 ++++-- run_fuzz_tests.sh | 5 +++-- test/fuzz_api_submissions.py | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 94a2a987..35563f27 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -71,7 +71,9 @@ jobs: - name: Fuzz tests run: | set +x - ./run_fuzz_tests.sh + ./run_fuzz_tests.sh 2>&1 > fuzz.log + echo "Preview test output:" + grep -A 1 -B 20 "Number of tests" fuzz.log - name: "Upload logs" if: success() || failure() @@ -79,5 +81,5 @@ jobs: with: name: boofuzz-results path: | - boofuzz-results/*.{db} + fuzz.log if-no-files-found: warn diff --git a/run_fuzz_tests.sh b/run_fuzz_tests.sh index 96dc3244..21d737aa 100755 --- a/run_fuzz_tests.sh +++ b/run_fuzz_tests.sh @@ -34,6 +34,7 @@ echo "Running fuzz tests..." export CCF_HOST=${CCF_HOST:-"localhost"} export CCF_PORT=${CCF_PORT:-8000} export CCF_URL="https://${CCF_HOST}:${CCF_PORT}" +echo "Service URL: $CCF_URL" if [ "$DOCKER" = "1" ]; then echo "Will use a running docker instance for testing..." @@ -47,8 +48,8 @@ else echo "Will use a built SCITT binary for testing..." PLATFORM=$PLATFORM ./start.sh & - CCF_PROCESS_PID=$! - trap "kill $CCF_PROCESS_PID" EXIT + # start script will launch cchost process + trap 'pkill -f cchost' EXIT export CCF_URL="https://localhost:8000" wait_for_service "$CCF_URL/node/network" diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index c159a587..8189d982 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -35,7 +35,8 @@ def checker(target, fuzz_data_logger, session, *args, **kwargs): is_success = False if time.time() - start_time_sec > test_time_threshold_sec: - fuzz_data_logger.log_info("Timeout reached") + fuzz_data_logger.log_info("Timeout reached: {} seconds".format(test_time_threshold_sec)) + fuzz_data_logger.log_info("Started at: {} and now is: {}".format(start_time_sec, time.time())) session._index_end = ( 0 # stop fuzzing https://github.com/jtpereyda/boofuzz/discussions/600 ) From 6fa08ff780a6d8219233f0e07bd9575a04ed48ca Mon Sep 17 00:00:00 2001 From: Ivar Date: Wed, 11 Dec 2024 16:14:24 +0000 Subject: [PATCH 19/19] reformat --- test/fuzz_api_submissions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/fuzz_api_submissions.py b/test/fuzz_api_submissions.py index 8189d982..6d4896b1 100644 --- a/test/fuzz_api_submissions.py +++ b/test/fuzz_api_submissions.py @@ -35,8 +35,12 @@ def checker(target, fuzz_data_logger, session, *args, **kwargs): is_success = False if time.time() - start_time_sec > test_time_threshold_sec: - fuzz_data_logger.log_info("Timeout reached: {} seconds".format(test_time_threshold_sec)) - fuzz_data_logger.log_info("Started at: {} and now is: {}".format(start_time_sec, time.time())) + fuzz_data_logger.log_info( + "Timeout reached: {} seconds".format(test_time_threshold_sec) + ) + fuzz_data_logger.log_info( + "Started at: {} and now is: {}".format(start_time_sec, time.time()) + ) session._index_end = ( 0 # stop fuzzing https://github.com/jtpereyda/boofuzz/discussions/600 )