Skip to content

Commit 49ea5c0

Browse files
nicallennicallen
authored
fix: make build script check fail when no repo is found (#699)
Change build script check to fail on skip instead of pass, which became incorrect when the check dependency relationship was changed so that the build script check is skipped when the version control system check fails (and thus should likewise fail as no source or build script is available), rather than being skipped when the stronger build checks pass (which previously meant that build script would then pass by default). Added an integration test to confirm that all checks fail for a purl referring to a nonexistent artifact. Signed-off-by: Nicholas Allen <nicholas.allen@oracle.com>
1 parent 7fc043e commit 49ea5c0

File tree

3 files changed

+238
-1
lines changed

3 files changed

+238
-1
lines changed

‎scripts/dev_scripts/integration_tests.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,15 @@ $RUN_MACARON analyze -purl pkg:private_domain.com/apache/maven -sbom "$SBOM_FILE
268268

269269
check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
270270

271+
echo -e "\n----------------------------------------------------------------------------------"
272+
echo "com.example/nonexistent: Analyzing purl of nonexistent artifact."
273+
echo -e "----------------------------------------------------------------------------------\n"
274+
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json
275+
JSON_RESULT=$WORKSPACE/output/reports/maven/com_example/nonexistent/nonexistent.json
276+
$RUN_MACARON analyze -purl pkg:maven/com.example/nonexistent@1.0.0 --skip-deps || log_fail
277+
278+
check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
279+
271280
# Analyze micronaut-projects/micronaut-test.
272281
echo -e "\n=================================================================================="
273282
echo "Run integration tests with configurations for micronaut-projects/micronaut-test..."

‎src/macaron/slsa_analyzer/checks/build_script_check.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ def __init__(self) -> None:
8181
description=description,
8282
depends_on=depends_on,
8383
eval_reqs=eval_reqs,
84-
result_on_skip=CheckResultType.PASSED,
84+
result_on_skip=CheckResultType.FAILED,
8585
)
8686

8787
def run_check(self, ctx: AnalyzeContext) -> CheckResultData:

‎tests/e2e/expected_results/purl/maven/com_example_nonexistent/nonexistent.json‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
{
2+
"metadata": {
3+
"timestamps": "2024-04-11 10:55:11",
4+
"has_passing_check": false,
5+
"run_checks": [
6+
"mcn_provenance_available_1",
7+
"mcn_build_as_code_1",
8+
"mcn_provenance_witness_level_one_1",
9+
"mcn_trusted_builder_level_three_1",
10+
"mcn_build_service_1",
11+
"mcn_provenance_expectation_1",
12+
"mcn_infer_artifact_pipeline_1",
13+
"mcn_build_script_1",
14+
"mcn_provenance_level_three_1",
15+
"mcn_version_control_system_1"
16+
],
17+
"check_tree": {
18+
"mcn_provenance_available_1": {
19+
"mcn_provenance_level_three_1": {},
20+
"mcn_provenance_expectation_1": {},
21+
"mcn_provenance_witness_level_one_1": {}
22+
},
23+
"mcn_version_control_system_1": {
24+
"mcn_build_script_1": {},
25+
"mcn_trusted_builder_level_three_1": {
26+
"mcn_build_as_code_1": {
27+
"mcn_infer_artifact_pipeline_1": {},
28+
"mcn_build_service_1": {}
29+
}
30+
}
31+
}
32+
}
33+
},
34+
"target": {
35+
"info": {
36+
"full_name": "pkg:maven/com.example/nonexistent@1.0.0",
37+
"local_cloned_path": "Unable to find a repository.",
38+
"remote_path": "",
39+
"branch": "",
40+
"commit_hash": "",
41+
"commit_date": ""
42+
},
43+
"provenances": {
44+
"is_inferred": true,
45+
"content": {}
46+
},
47+
"checks": {
48+
"summary": {
49+
"DISABLED": 0,
50+
"FAILED": 10,
51+
"PASSED": 0,
52+
"SKIPPED": 0,
53+
"UNKNOWN": 0
54+
},
55+
"results": [
56+
{
57+
"check_id": "mcn_build_script_1",
58+
"check_description": "Check if the target repo has a valid build script.",
59+
"slsa_requirements": [
60+
"Scripted Build - SLSA Level 1"
61+
],
62+
"justification": [
63+
"Not Available."
64+
],
65+
"result_type": "FAILED"
66+
},
67+
{
68+
"check_id": "mcn_build_as_code_1",
69+
"check_description": "The build definition and configuration executed by the build service is verifiably derived from text file definitions stored in a version control system.",
70+
"slsa_requirements": [
71+
"Build as code - SLSA Level 3"
72+
],
73+
"justification": [
74+
"Not Available."
75+
],
76+
"result_type": "FAILED"
77+
},
78+
{
79+
"check_id": "mcn_build_service_1",
80+
"check_description": "Check if the target repo has a valid build service.",
81+
"slsa_requirements": [
82+
"Build service - SLSA Level 2"
83+
],
84+
"justification": [
85+
"Not Available."
86+
],
87+
"result_type": "FAILED"
88+
},
89+
{
90+
"check_id": "mcn_infer_artifact_pipeline_1",
91+
"check_description": "Detects potential pipelines from which an artifact is published.",
92+
"slsa_requirements": [
93+
"Build as code - SLSA Level 3"
94+
],
95+
"justification": [
96+
"Not Available."
97+
],
98+
"result_type": "FAILED"
99+
},
100+
{
101+
"check_id": "mcn_provenance_available_1",
102+
"check_description": "Check whether the target has intoto provenance.",
103+
"slsa_requirements": [
104+
"Provenance - Available - SLSA Level 1",
105+
"Provenance content - Identifies build instructions - SLSA Level 1",
106+
"Provenance content - Identifies artifacts - SLSA Level 1",
107+
"Provenance content - Identifies builder - SLSA Level 1"
108+
],
109+
"justification": [
110+
"Not Available."
111+
],
112+
"result_type": "FAILED"
113+
},
114+
{
115+
"check_id": "mcn_provenance_expectation_1",
116+
"check_description": "Check whether the SLSA provenance for the produced artifact conforms to the expected value.",
117+
"slsa_requirements": [
118+
"Provenance conforms with expectations - SLSA Level 3"
119+
],
120+
"justification": [
121+
"Not Available."
122+
],
123+
"result_type": "FAILED"
124+
},
125+
{
126+
"check_id": "mcn_provenance_level_three_1",
127+
"check_description": "Check whether the target has SLSA provenance level 3.",
128+
"slsa_requirements": [
129+
"Provenance - Non falsifiable - SLSA Level 3",
130+
"Provenance content - Includes all build parameters - SLSA Level 3",
131+
"Provenance content - Identifies entry point - SLSA Level 3",
132+
"Provenance content - Identifies source code - SLSA Level 2"
133+
],
134+
"justification": [
135+
"Not Available."
136+
],
137+
"result_type": "FAILED"
138+
},
139+
{
140+
"check_id": "mcn_provenance_witness_level_one_1",
141+
"check_description": "Check whether the target has a level-1 witness provenance.",
142+
"slsa_requirements": [
143+
"Provenance - Available - SLSA Level 1",
144+
"Provenance content - Identifies build instructions - SLSA Level 1",
145+
"Provenance content - Identifies artifacts - SLSA Level 1",
146+
"Provenance content - Identifies builder - SLSA Level 1"
147+
],
148+
"justification": [
149+
"Not Available."
150+
],
151+
"result_type": "FAILED"
152+
},
153+
{
154+
"check_id": "mcn_trusted_builder_level_three_1",
155+
"check_description": "Check whether the target uses a trusted SLSA level 3 builder.",
156+
"slsa_requirements": [
157+
"Hermetic - SLSA Level 4",
158+
"Isolated - SLSA Level 3",
159+
"Parameterless - SLSA Level 4",
160+
"Ephemeral environment - SLSA Level 3"
161+
],
162+
"justification": [
163+
"Not Available."
164+
],
165+
"result_type": "FAILED"
166+
},
167+
{
168+
"check_id": "mcn_version_control_system_1",
169+
"check_description": "Check whether the target repo uses a version control system.",
170+
"slsa_requirements": [
171+
"Version controlled - SLSA Level 2"
172+
],
173+
"justification": [
174+
"Not Available."
175+
],
176+
"result_type": "FAILED"
177+
}
178+
]
179+
}
180+
},
181+
"dependencies": {
182+
"analyzed_deps": 0,
183+
"unique_dep_repos": 0,
184+
"checks_summary": [
185+
{
186+
"check_id": "mcn_provenance_available_1",
187+
"num_deps_pass": 0
188+
},
189+
{
190+
"check_id": "mcn_build_as_code_1",
191+
"num_deps_pass": 0
192+
},
193+
{
194+
"check_id": "mcn_provenance_witness_level_one_1",
195+
"num_deps_pass": 0
196+
},
197+
{
198+
"check_id": "mcn_trusted_builder_level_three_1",
199+
"num_deps_pass": 0
200+
},
201+
{
202+
"check_id": "mcn_build_service_1",
203+
"num_deps_pass": 0
204+
},
205+
{
206+
"check_id": "mcn_provenance_expectation_1",
207+
"num_deps_pass": 0
208+
},
209+
{
210+
"check_id": "mcn_infer_artifact_pipeline_1",
211+
"num_deps_pass": 0
212+
},
213+
{
214+
"check_id": "mcn_build_script_1",
215+
"num_deps_pass": 0
216+
},
217+
{
218+
"check_id": "mcn_provenance_level_three_1",
219+
"num_deps_pass": 0
220+
},
221+
{
222+
"check_id": "mcn_version_control_system_1",
223+
"num_deps_pass": 0
224+
}
225+
],
226+
"dep_status": []
227+
}
228+
}

0 commit comments

Comments
 (0)