Skip to content

chore: convert expectation files to policies and use in integration tests #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jun 13, 2024
Merged

chore: convert expectation files to policies and use in integration tests #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
004c660
chore: convert expectation files to policies and use in integration t…
benmss Jun 6, 2024
223e0bd
chore: prefer is_repo rule in converted policies
benmss Jun 10, 2024
fbed57f
chore: update integration test function name; remove added dependency…
benmss Jun 10, 2024
ec85d07
chore: use simplified rule for repositories
benmss Jun 10, 2024
d4bcd73
chore: minor fix
benmss Jun 10, 2024
8f3b327
chore: fix policy conflicts with vsa tests
benmss Jun 11, 2024
5d0e10b
chore: ensure repository table uses URL not PURL; apply changes to do…
benmss Jun 12, 2024
6e9f113
chore: remove leftover JSON files
benmss Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
343 changes: 156 additions & 187 deletions scripts/dev_scripts/integration_tests.sh
View file
Open in desktop

Large diffs are not rendered by default.

94 changes: 46 additions & 48 deletions scripts/dev_scripts/integration_tests_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,19 @@ RUN_MACARON_SCRIPT=$2

# The scripts to compare the results of the integration tests.
COMPARE_DEPS=$WORKSPACE/tests/dependency_analyzer/compare_dependencies.py
COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py
COMPARE_POLICIES=$WORKSPACE/tests/policy_engine/compare_policy_reports.py
COMPARE_VSA=$WORKSPACE/tests/vsa/compare_vsa.py
UNIT_TEST_SCRIPT=$WORKSPACE/scripts/dev_scripts/test_run_macaron_sh.py
RUN_POLICY="$RUN_MACARON_SCRIPT verify-policy"
DB=$WORKSPACE/output/macaron.db

RESULT_CODE=0

function run_macaron_clean() {
rm $DB
$RUN_MACARON_SCRIPT "$@"
}

function log_fail() {
printf "Error: FAILED integration test (line ${BASH_LINENO}) %s\n" $@
RESULT_CODE=1
Expand All @@ -37,49 +43,47 @@ echo "with dependency resolution using cyclonedx Gradle plugin (default)."
echo -e "----------------------------------------------------------------------------------\n"
DEP_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/dependencies.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_timyarkov_multibuild_test.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/timyarkov/multibuild_test/multibuild_test.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.json
$RUN_MACARON_SCRIPT analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail
OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/multibuild_test/multibuild_test.dl
run_macaron_clean analyze -rp https://github.com/timyarkov/multibuild_test -b main -d a8b0efe24298bc81f63217aaa84776c3d48976c5 || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "apache/maven: Check the resolved dependency output with config for cyclonedx maven plugin (default)."
echo -e "----------------------------------------------------------------------------------\n"
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/cyclonedx_apache_maven.json

$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
run_macaron_clean analyze -c $WORKSPACE/tests/dependency_analyzer/configurations/maven_config.yaml || log_fail
python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "apache/maven: e2e using the local repo path, the branch name and the commit digest without dependency resolution."
echo -e "----------------------------------------------------------------------------------\n"
JSON_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/maven.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/maven/maven.json
OUTPUT_POLICY=$WORKSPACE/tests/e2e/expected_results/maven/maven.dl

run_macaron_clean -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

$RUN_MACARON_SCRIPT -lr $WORKSPACE/output/git_repos/github_com analyze -r apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "apache/maven: Check the e2e output JSON file with config and no dependency analyzing."
echo -e "----------------------------------------------------------------------------------\n"
JSON_RESULT_DIR=$WORKSPACE/output/reports/github_com/apache/maven
JSON_EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven
EXPECT_DIR=$WORKSPACE/tests/e2e/expected_results/maven

declare -a COMPARE_FILES=(
"maven.json"
"guava.json"
"mockito.json"
"maven.dl"
"guava.dl"
"mockito.dl"
)

$RUN_MACARON_SCRIPT analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail
run_macaron_clean analyze -c $WORKSPACE/tests/e2e/configurations/maven_config.yaml --skip-deps || log_fail

for i in "${COMPARE_FILES[@]}"
do
python $COMPARE_JSON_OUT $JSON_RESULT_DIR/$i $JSON_EXPECT_DIR/$i || log_fail
$RUN_POLICY -d $DB -f $EXPECT_DIR/$i || log_fail
done

echo -e "\n----------------------------------------------------------------------------------"
Expand All @@ -89,63 +93,47 @@ SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/apache_maven_
DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/apache_maven_with_sbom_provided.json
DEP_RESULT=$WORKSPACE/output/reports/github_com/apache/maven/dependencies.json

$RUN_MACARON_SCRIPT analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail
run_macaron_clean analyze -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b -sbom $SBOM_FILE || log_fail

python $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "apache/maven: Analyzing with PURL and repository path without dependency resolution."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.json
JSON_RESULT=$WORKSPACE/output/reports/maven/apache/maven/maven.json
$RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/purl/maven/maven.dl
run_macaron_clean analyze -purl pkg:maven/apache/maven -rp https://github.com/apache/maven -b master -d 3fc399318edef0d5ba593723a24fff64291d6f9b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
echo "The CUE expectation file is provided as a single file path."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/urllib3_PASS.cue
$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
run_macaron_clean analyze -pe $EXPECTATION_FILE -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
echo "The CUE expectation file should be found via the directory path."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/urllib3/urllib3/urllib3.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/urllib3/urllib3.dl
EXPECTATION_DIR=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/
$RUN_MACARON_SCRIPT analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail
run_macaron_clean analyze -pe $EXPECTATION_DIR -rp https://github.com/urllib3/urllib3/urllib3 -b main -d 87a0ecee6e691fe5ff93cd000c0158deebef763b --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "Test verifying CUE provenance expectation for ossf/scorecard"
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.json
JSON_RESULT=$WORKSPACE/output/reports/github/ossf/scorecard/scorecard.json
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/scorecard/scorecard.dl
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/scorecard.ini
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/scorecard_PASS.cue
$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail
run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -purl pkg:github/ossf/scorecard@v4.13.1 --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
echo "and CUE file is provided as expectation."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json
JSON_RESULT=$WORKSPACE/output/reports/github_com/slsa-framework/slsa-verifier/slsa-verifier.json
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
$RUN_MACARON_SCRIPT -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail

python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "Run policy CLI with scorecard results."
Expand All @@ -160,11 +148,21 @@ $RUN_MACARON_SCRIPT verify-policy -f $POLICY_FILE -d "$WORKSPACE/output/macaron.
python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
python "$COMPARE_VSA" "$VSA_RESULT" "$VSA_PAYLOAD_EXPECTED" || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "slsa-framework/slsa-verifier: Analyzing the repo path when automatic dependency resolution is skipped"
echo "and CUE file is provided as expectation."
echo -e "----------------------------------------------------------------------------------\n"
JSON_EXPECTED=$WORKSPACE/tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.dl
EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/slsa_verifier_PASS.cue
DEFAULTS_FILE=$WORKSPACE/tests/e2e/defaults/slsa_verifier.ini
run_macaron_clean -dp $DEFAULTS_FILE analyze -pe $EXPECTATION_FILE -rp https://github.com/slsa-framework/slsa-verifier -b main -d fc50b662fcfeeeb0e97243554b47d9b20b14efac --skip-deps || log_fail

$RUN_POLICY -d $DB -f $OUTPUT_POLICY || log_fail

echo -e "\n----------------------------------------------------------------------------------"
echo "behnazh-w/example-maven-app as a local and remote repository"
echo "Test the Witness and GitHub provenances as an input, Cue expectation validation, Policy CLI and VSA generation."
echo -e "----------------------------------------------------------------------------------\n"
RUN_POLICY="macaron verify-policy"
POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/example-maven-project/policy.dl
POLICY_RESULT=$WORKSPACE/output/policy_report.json
POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/example-maven-project/example_maven_project_policy_report.json
Expand All @@ -179,7 +177,7 @@ WITNESS_PROVENANCE_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/resources/vali
git clone https://github.com/behnazh-w/example-maven-app.git $WORKSPACE/output/git_repos/local_repos/example-maven-app || log_fail

# Check the Witness provenance.
$RUN_MACARON_SCRIPT analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail
run_macaron_clean analyze -pf $WITNESS_PROVENANCE_FILE -pe $WITNESS_EXPECTATION_FILE -purl pkg:maven/io.github.behnazh-w.demo/example-maven-app@1.0-SNAPSHOT?type=jar --repo-path example-maven-app --skip-deps || log_fail

# Test the remote repo with GitHub provenance.
GITHUB_EXPECTATION_FILE=$WORKSPACE/tests/slsa_analyzer/provenance/expectations/cue/resources/valid_expectations/github-example-maven-project.cue
Expand Down
11 changes: 10 additions & 1 deletion src/macaron/policy_engine/prelude/helper_rules.dl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright (c) 2023 - 2023, Oracle and/or its affiliates. All rights reserved. */
/* Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. */
/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */

/* Souffle datalog rules to assist in policy authoring */
Expand All @@ -9,6 +9,9 @@
.decl check_passed(component_id: number, check_name: symbol)
check_passed(component_id, check_name) :- check_result(_, check_name, 1, component_id).

.decl check_failed(component_id: number, check_name: symbol)
check_failed(component_id, check_name) :- check_result(_, check_name, 0, component_id).

/**
* Check name is valid.
*/
Expand Down Expand Up @@ -42,6 +45,12 @@ transitive_dependency(parent, dependency) :-
.decl is_repo(repo_id: number, repo_complete_name: symbol, component_id: number)
is_repo(repo_id, repo_complete_name, component_id) :- repository(repo_id, component_id, repo_complete_name, _, _, _, _, _, _, _, _, _, _).

/**
* Extract the id and URL from the repository relation.
*/
.decl is_repo_url(repo_id: number, url: symbol)
is_repo_url(repo_id, url) :- repository(_, repo_id, _, _, _, _, _, url, _, _, _, _, _).


/**
* A convenience relation to find analyses conducted on a specific repository.
Expand Down
22 changes: 22 additions & 0 deletions tests/e2e/expected_results/docker_test/docker_test.dl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */
/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */

#include "prelude.dl"

Policy("test_policy", component_id, "") :-
check_passed(component_id, "mcn_build_as_code_1"),
check_passed(component_id, "mcn_build_script_1"),
check_passed(component_id, "mcn_build_service_1"),
check_passed(component_id, "mcn_version_control_system_1"),
check_failed(component_id, "mcn_infer_artifact_pipeline_1"),
check_failed(component_id, "mcn_provenance_available_1"),
check_failed(component_id, "mcn_provenance_derived_commit_1"),
check_failed(component_id, "mcn_provenance_derived_repo_1"),
check_failed(component_id, "mcn_provenance_expectation_1"),
check_failed(component_id, "mcn_provenance_level_three_1"),
check_failed(component_id, "mcn_provenance_witness_level_one_1"),
check_failed(component_id, "mcn_trusted_builder_level_three_1"),
is_repo_url(component_id, "https://github.com/timyarkov/docker_test").

apply_policy_to("test_policy", component_id) :-
is_component(component_id, "pkg:github.com/timyarkov/docker_test@404a51a2f38c4470af6b32e4e00b5318c2d7c0cc").
Loading