diff --git a/scripts/dev_scripts/integration_tests.sh b/scripts/dev_scripts/integration_tests.sh index a6dfb11c9..452555651 100755 --- a/scripts/dev_scripts/integration_tests.sh +++ b/scripts/dev_scripts/integration_tests.sh @@ -268,6 +268,15 @@ $RUN_MACARON analyze -purl pkg:private_domain.com/apache/maven -sbom "$SBOM_FILE check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail +echo -e "\n----------------------------------------------------------------------------------" +echo "com.example/nonexistent: Analyzing purl of nonexistent artifact." +echo -e "----------------------------------------------------------------------------------\n" +JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json +JSON_RESULT=$WORKSPACE/output/reports/maven/com_example/nonexistent/nonexistent.json +$RUN_MACARON analyze -purl pkg:maven/com.example/nonexistent@1.0.0 --skip-deps || log_fail + +check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail + # Analyze micronaut-projects/micronaut-test. echo -e "\n==================================================================================" echo "Run integration tests with configurations for micronaut-projects/micronaut-test..." diff --git a/src/macaron/slsa_analyzer/checks/build_script_check.py b/src/macaron/slsa_analyzer/checks/build_script_check.py index 2f355bd99..80b8bdd6b 100644 --- a/src/macaron/slsa_analyzer/checks/build_script_check.py +++ b/src/macaron/slsa_analyzer/checks/build_script_check.py @@ -81,7 +81,7 @@ def __init__(self) -> None: description=description, depends_on=depends_on, eval_reqs=eval_reqs, - result_on_skip=CheckResultType.PASSED, + result_on_skip=CheckResultType.FAILED, ) def run_check(self, ctx: AnalyzeContext) -> CheckResultData: diff --git a/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json b/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json new file mode 100644 index 000000000..bbef939dd --- /dev/null +++ b/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json @@ -0,0 +1,228 @@ +{ + "metadata": { + "timestamps": "2024-04-11 10:55:11", + "has_passing_check": false, + "run_checks": [ + "mcn_provenance_available_1", + "mcn_build_as_code_1", + "mcn_provenance_witness_level_one_1", + "mcn_trusted_builder_level_three_1", + "mcn_build_service_1", + "mcn_provenance_expectation_1", + "mcn_infer_artifact_pipeline_1", + "mcn_build_script_1", + "mcn_provenance_level_three_1", + "mcn_version_control_system_1" + ], + "check_tree": { + "mcn_provenance_available_1": { + "mcn_provenance_level_three_1": {}, + "mcn_provenance_expectation_1": {}, + "mcn_provenance_witness_level_one_1": {} + }, + "mcn_version_control_system_1": { + "mcn_build_script_1": {}, + "mcn_trusted_builder_level_three_1": { + "mcn_build_as_code_1": { + "mcn_infer_artifact_pipeline_1": {}, + "mcn_build_service_1": {} + } + } + } + } + }, + "target": { + "info": { + "full_name": "pkg:maven/com.example/nonexistent@1.0.0", + "local_cloned_path": "Unable to find a repository.", + "remote_path": "", + "branch": "", + "commit_hash": "", + "commit_date": "" + }, + "provenances": { + "is_inferred": true, + "content": {} + }, + "checks": { + "summary": { + "DISABLED": 0, + "FAILED": 10, + "PASSED": 0, + "SKIPPED": 0, + "UNKNOWN": 0 + }, + "results": [ + { + "check_id": "mcn_build_script_1", + "check_description": "Check if the target repo has a valid build script.", + "slsa_requirements": [ + "Scripted Build - SLSA Level 1" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_build_as_code_1", + "check_description": "The build definition and configuration executed by the build service is verifiably derived from text file definitions stored in a version control system.", + "slsa_requirements": [ + "Build as code - SLSA Level 3" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_build_service_1", + "check_description": "Check if the target repo has a valid build service.", + "slsa_requirements": [ + "Build service - SLSA Level 2" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_infer_artifact_pipeline_1", + "check_description": "Detects potential pipelines from which an artifact is published.", + "slsa_requirements": [ + "Build as code - SLSA Level 3" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_provenance_available_1", + "check_description": "Check whether the target has intoto provenance.", + "slsa_requirements": [ + "Provenance - Available - SLSA Level 1", + "Provenance content - Identifies build instructions - SLSA Level 1", + "Provenance content - Identifies artifacts - SLSA Level 1", + "Provenance content - Identifies builder - SLSA Level 1" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_provenance_expectation_1", + "check_description": "Check whether the SLSA provenance for the produced artifact conforms to the expected value.", + "slsa_requirements": [ + "Provenance conforms with expectations - SLSA Level 3" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_provenance_level_three_1", + "check_description": "Check whether the target has SLSA provenance level 3.", + "slsa_requirements": [ + "Provenance - Non falsifiable - SLSA Level 3", + "Provenance content - Includes all build parameters - SLSA Level 3", + "Provenance content - Identifies entry point - SLSA Level 3", + "Provenance content - Identifies source code - SLSA Level 2" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_provenance_witness_level_one_1", + "check_description": "Check whether the target has a level-1 witness provenance.", + "slsa_requirements": [ + "Provenance - Available - SLSA Level 1", + "Provenance content - Identifies build instructions - SLSA Level 1", + "Provenance content - Identifies artifacts - SLSA Level 1", + "Provenance content - Identifies builder - SLSA Level 1" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_trusted_builder_level_three_1", + "check_description": "Check whether the target uses a trusted SLSA level 3 builder.", + "slsa_requirements": [ + "Hermetic - SLSA Level 4", + "Isolated - SLSA Level 3", + "Parameterless - SLSA Level 4", + "Ephemeral environment - SLSA Level 3" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + }, + { + "check_id": "mcn_version_control_system_1", + "check_description": "Check whether the target repo uses a version control system.", + "slsa_requirements": [ + "Version controlled - SLSA Level 2" + ], + "justification": [ + "Not Available." + ], + "result_type": "FAILED" + } + ] + } + }, + "dependencies": { + "analyzed_deps": 0, + "unique_dep_repos": 0, + "checks_summary": [ + { + "check_id": "mcn_provenance_available_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_build_as_code_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_witness_level_one_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_trusted_builder_level_three_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_build_service_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_expectation_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_infer_artifact_pipeline_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_build_script_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_provenance_level_three_1", + "num_deps_pass": 0 + }, + { + "check_id": "mcn_version_control_system_1", + "num_deps_pass": 0 + } + ], + "dep_status": [] + } +}