feat: add dependency resolution for Python

Signed-off-by: behnazh-w <behnaz.hassanshahi@oracle.com>

Author: behnazh-w

docker/user.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 # We update the GID and UID of the existing macaron user in the container
@@ -52,6 +52,24 @@ then
     mkdir --parents "$HOME"/output
 fi
 
+# Prepare the python virtual environment. We copy the mounted directory to `.python_venv` to fix the symbolic
+# links to the Python interpreter in the container without affecting the files on host.
+if [[ -d "$HOME/python_venv" ]];
+then
+    cp -r "$HOME/python_venv" "$HOME/.python_venv"
+fi
+python_binaries=(
+    "python"
+    "python3"
+    "python3.11"
+)
+for p in "${python_binaries[@]}"; do
+    if [[ -f "$HOME/.venv/bin/${p}" ]];
+    then
+        ln -sf "$HOME/.venv/bin/${p}" "$HOME/.python_venv/bin/${p}"
+    fi
+done
+
 # The directory that could be mounted to the host machine file systems should
 # have the owner as the current user in the host machine.
 chown --recursive macaron:macaron "$HOME"/.m2

docs/source/pages/developers_guide/apidoc/macaron.dependency_analyzer.rst

@@ -33,10 +33,10 @@ macaron.dependency\_analyzer.cyclonedx\_mvn module
    :undoc-members:
    :show-inheritance:
 
-macaron.dependency\_analyzer.dependency\_resolver module
---------------------------------------------------------
+macaron.dependency\_analyzer.cyclonedx\_python module
+-----------------------------------------------------
 
-.. automodule:: macaron.dependency_analyzer.dependency_resolver
+.. automodule:: macaron.dependency_analyzer.cyclonedx_python
    :members:
    :undoc-members:
    :show-inheritance:

pyproject.toml

@@ -31,7 +31,9 @@ dependencies = [
     "defusedxml >=0.7.1,<1.0.0",
     "packageurl-python >= 0.11.1,<1.0.0",
     "ruamel.yaml >= 0.18.6,<1.0.0",
-    "jsonschema >= 4.22.0,<5.0.0"
+    "jsonschema >= 4.22.0,<5.0.0",
+    "cyclonedx-bom >=4.0.0,<5.0.0",
+    "cyclonedx-python-lib >=7.3.4,<8.0.0",
 ]
 keywords = []
 # https://pypi.org/classifiers/

scripts/dev_scripts/integration_tests.sh

@@ -14,6 +14,7 @@ COMPARE_VSA=$WORKSPACE/tests/vsa/compare_vsa.py
 TEST_REPO_FINDER=$WORKSPACE/tests/e2e/repo_finder/repo_finder.py
 TEST_COMMIT_FINDER=$WORKSPACE/tests/e2e/repo_finder/commit_finder.py
 RUN_MACARON="python -m macaron -o $WORKSPACE/output"
+MAKE_VENV="python -m venv"
 RESULT_CODE=0
 UPDATE=0
 
@@ -262,9 +263,9 @@ echo "apache/maven: Analyzing using a CycloneDx SBOM file of a software componen
 echo -e "----------------------------------------------------------------------------------\n"
 SBOM_FILE=$WORKSPACE/tests/dependency_analyzer/cyclonedx/resources/private_mirror_apache_maven.json
 DEP_EXPECTED=$WORKSPACE/tests/dependency_analyzer/expected_results/private_mirror_apache_maven.json
-DEP_RESULT=$WORKSPACE/output/reports/private-domain_com/apache/maven/dependencies.json
+DEP_RESULT=$WORKSPACE/output/reports/maven/private_apache_maven/maven/dependencies.json
 
-$RUN_MACARON analyze -purl pkg:private-domain.com/apache/maven -sbom "$SBOM_FILE" || log_fail
+$RUN_MACARON analyze -purl pkg:maven/private.apache.maven/maven@4.0.0-alpha-1-SNAPSHOT?type=pom -sbom "$SBOM_FILE" || log_fail
 
 check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
 
@@ -457,6 +458,27 @@ $RUN_MACARON -lr $WORKSPACE/output/git_repos/github_com analyze -rp apache/maven
 check_or_update_expected_output $COMPARE_DEPS $DEP_RESULT $DEP_EXPECTED || log_fail
 check_or_update_expected_output $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with virtual env provided as input."
+echo -e "----------------------------------------------------------------------------------\n"
+# Prepare the virtual environment.
+VIRTUAL_ENV_PATH=$WORKSPACE/.django_venv
+$MAKE_VENV "$VIRTUAL_ENV_PATH"
+"$VIRTUAL_ENV_PATH"/bin/pip install django==5.0.6
+$RUN_MACARON analyze -purl pkg:pypi/django@5.0.6 --python-venv "$VIRTUAL_ENV_PATH" || log_fail
+
+# Check the dependencies using the policy engine.
+RUN_POLICY="macaron verify-policy"
+POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/django/test_dependencies.dl
+POLICY_RESULT=$WORKSPACE/output/policy_report.json
+POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/django/test_dependencies.json
+
+$RUN_POLICY -f "$POLICY_FILE" -d "$WORKSPACE/output/macaron.db" || log_fail
+check_or_update_expected_output $COMPARE_POLICIES "$POLICY_RESULT" "$POLICY_EXPECTED" || log_fail
+
+# Clean up and remove the virtual environment.
+rm -rf "$VIRTUAL_ENV_PATH"
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "apache/maven: Analyzing with local paths in configuration and without dependency resolution."
 echo -e "----------------------------------------------------------------------------------\n"

scripts/dev_scripts/integration_tests_docker.sh

@@ -18,6 +18,7 @@ COMPARE_JSON_OUT=$WORKSPACE/tests/e2e/compare_e2e_result.py
 COMPARE_POLICIES=$WORKSPACE/tests/policy_engine/compare_policy_reports.py
 COMPARE_VSA=$WORKSPACE/tests/vsa/compare_vsa.py
 UNIT_TEST_SCRIPT=$WORKSPACE/scripts/dev_scripts/test_run_macaron_sh.py
+MAKE_VENV="python -m venv"
 
 RESULT_CODE=0
 
@@ -102,6 +103,26 @@ $RUN_MACARON_SCRIPT analyze -purl pkg:maven/apache/maven -rp https://github.com/
 
 python $COMPARE_JSON_OUT $JSON_RESULT $JSON_EXPECTED || log_fail
 
+echo -e "\n----------------------------------------------------------------------------------"
+echo "pkg:pypi/django@5.0.6: Analyzing the dependencies with virtual env provided as input."
+echo -e "----------------------------------------------------------------------------------\n"
+# Prepare the virtual environment.
+VIRTUAL_ENV_PATH=$WORKSPACE/.django_venv
+$MAKE_VENV "$VIRTUAL_ENV_PATH"
+"$VIRTUAL_ENV_PATH"/bin/pip install django==5.0.6
+$RUN_MACARON_SCRIPT analyze -purl pkg:pypi/django@5.0.6 --python-venv "$VIRTUAL_ENV_PATH" || log_fail
+
+# Check the dependencies using the policy engine.
+POLICY_FILE=$WORKSPACE/tests/policy_engine/resources/policies/django/test_dependencies.dl
+POLICY_RESULT=$WORKSPACE/output/policy_report.json
+POLICY_EXPECTED=$WORKSPACE/tests/policy_engine/expected_results/django/test_dependencies.json
+
+$RUN_MACARON_SCRIPT verify-policy -f "$POLICY_FILE" -d "$WORKSPACE/output/macaron.db" || log_fail
+python $COMPARE_POLICIES $POLICY_RESULT $POLICY_EXPECTED || log_fail
+
+# Clean up and remove the virtual environment.
+rm -rf "$VIRTUAL_ENV_PATH"
+
 echo -e "\n----------------------------------------------------------------------------------"
 echo "urllib3/urllib3: Analyzing the repo path when automatic dependency resolution is skipped."
 echo "The CUE expectation file is provided as a single file path."

scripts/release_scripts/run_macaron.sh

@@ -327,6 +327,10 @@ if [[ $command == "analyze" ]]; then
                 arg_template_path="$2"
                 shift
                 ;;
+            --python-venv)
+                python_venv_path="$2"
+                shift
+                ;;
             *)
                 rest_command+=("$1")
                 ;;
@@ -454,6 +458,16 @@ if [[ -n "${arg_prov_file:-}" ]]; then
     mount_file "-pf/--provenance-file" "$prov_file_path" "$prov_file_path_in_container" "ro,Z"
 fi
 
+# Mount the Python virtual environment into ${MACARON_WORKSPACE}/python_venv.
+if [[ -n "${python_venv_path:-}" ]]; then
+    python_venv_in_container="${MACARON_WORKSPACE}/python_venv"
+    # We copy the mounted directory to `.python_venv` once the container starts running to
+    # be able to make changes to the mounted files without affecting the files on host.
+    argv_command+=("--python-venv" "${MACARON_WORKSPACE}/.python_venv")
+
+    mount_dir_ro "--python-venv" "$python_venv_path" "$python_venv_in_container"
+fi
+
 # MACARON entrypoint - verify-policy command argvs
 # This is for macaron verify-policy command.
 # Determine the database path to be mounted into ${MACARON_WORKSPACE}/database/macaron.db

src/macaron/__main__.py

@@ -57,6 +57,15 @@ def analyze_slsa_levels_single(analyzer_single_args: argparse.Namespace) -> None
             sys.exit(os.EX_OSFILE)
         global_config.load_expectation_files(analyzer_single_args.provenance_expectation)
 
+    # Set Python virtual environment path.
+    if analyzer_single_args.python_venv is not None:
+        if not os.path.exists(analyzer_single_args.python_venv):
+            logger.critical(
+                'The Python virtual environment path "%s" does not exist.', analyzer_single_args.python_venv
+            )
+            sys.exit(os.EX_OSFILE)
+        global_config.load_python_venv(analyzer_single_args.python_venv)
+
     analyzer = Analyzer(global_config.output_path, global_config.build_log_path)
 
     # Initiate reporters.
@@ -412,6 +421,12 @@ def main(argv: list[str] | None = None) -> None:
         help=("The path to the Jinja2 html template (please make sure to use .html or .j2 extensions)."),
     )
 
+    single_analyze_parser.add_argument(
+        "--python-venv",
+        required=False,
+        help=("The path to the Python virtual environment of the target software component."),
+    )
+
     # Dump the default values.
     sub_parser.add_parser(name="dump-defaults", description="Dumps the defaults.ini file to the output directory.")
 

src/macaron/config/defaults.ini

@@ -40,6 +40,10 @@ dep_tool_gradle = cyclonedx-gradle:1.7.4
 # This is the timeout (in seconds) to run the dependency resolver.
 timeout = 2400
 recursive = False
+# Determines whether the CycloneDX BOM file should be validated or not.
+validate = True
+# The CycloneDX schema version used for validation.
+schema = 1.6
 
 # This is the repo finder script.
 [repofinder]

src/macaron/config/global_config.py

@@ -13,17 +13,39 @@
 class GlobalConfig:
     """Class for keeping track of global configurations."""
 
+    #: The provenance expectation paths.
     expectation_paths: list[str] = field(default_factory=list)
+
+    #: The path to the Macaron Python package.
     macaron_path: str = ""
+
+    #: The path to the output files.
     output_path: str = ""
+
+    #: The path to build logs directory.
     build_log_path: str = ""
+
+    #: The path to the local clone of the target repository.
     local_repos_path: str = ""
+
+    #: The GitHub token.
     gh_token: str = ""
+
+    #: The GitLab public token.
     gl_token: str = ""
+
+    #: The GitLab self-hosted token.
     gl_self_host_token: str = ""
+
+    #: The debug level.
     debug_level: int = logging.DEBUG
+
+    #: The path to resources directory.
     resources_path: str = ""
 
+    #: The path to Python virtual environment.
+    python_venv_path: str = ""
+
     def load(
         self,
         macaron_path: str,
@@ -79,6 +101,20 @@ def load_expectation_files(self, exp_path: str) -> None:
 
         self.expectation_paths = exp_files
 
+    def load_python_venv(self, venv_path: str) -> None:
+        """
+        Load Python virtual environment.
+
+        Parameters
+        ----------
+        venv_path : str
+            The path to the Python virtual environment of the target software component.
+        """
+        if os.path.isdir(venv_path):
+            logger.info("Successfully loaded %s", venv_path)
+
+        self.python_venv_path = str(os.path.abspath(venv_path))
+
 
 global_config = GlobalConfig()
 """The object that can be imported and used globally."""

src/macaron/dependency_analyzer/__init__.py

@@ -1,12 +1,4 @@
 # Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
-"""This package contains the dependency resolvers for Java projects."""
-
-from .dependency_resolver import (  # noqa: F401
-    DependencyAnalyzer,
-    DependencyAnalyzerError,
-    DependencyInfo,
-    DependencyTools,
-    NoneDependencyAnalyzer,
-)
+"""This package contains the dependency resolvers."""