From c9c023e7712044475578d03814f34bb03435d949 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 7 Jan 2025 21:10:31 +0530 Subject: [PATCH] Fix conda environment yaml parsing errors Signed-off-by: Ayan Sinha Mahapatra --- src/packagedcode/conda.py | 3 + .../phc-gnn/environment_gpu.yml-expected.json | 2 +- .../ringer/environment.yaml-expected.json | 2 +- .../conda-yaml/test/environment_host_port.yml | 3 + .../environment_host_port.yml-expected.json | 58 +++++++++++++++++++ .../misc/conda/scenicplus.yaml-expected.json | 2 +- tests/packagedcode/test_conda.py | 6 ++ 7 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml create mode 100644 tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml-expected.json diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index 80b71cbb12..cbe4c1b353 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -51,6 +51,7 @@ def parse(cls, location, package_only=False): primary_language=cls.default_primary_language, dependencies=dependencies, extra_data=extra_data, + is_private=True, ) yield models.PackageData.from_data(package_data, package_only) @@ -205,6 +206,8 @@ def get_conda_yaml_dependencies(conda_data): if "::" in dep: namespace, dep = dep.split("::") + if "/" in namespace or ":" in namespace: + namespace = None req = parse_requirement_line(dep) if req: diff --git a/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json index 5ccbfdd8c0..70b2380e4d 100644 --- a/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json +++ b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json @@ -33,7 +33,7 @@ "notice_text": null, "source_packages": [], "file_references": [], - "is_private": false, + "is_private": true, "is_virtual": false, "extra_data": { "channels": [ diff --git a/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json index bcba7bb05c..c6f3a8f985 100644 --- a/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json +++ b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json @@ -33,7 +33,7 @@ "notice_text": null, "source_packages": [], "file_references": [], - "is_private": false, + "is_private": true, "is_virtual": false, "extra_data": { "channels": [ diff --git a/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml b/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml new file mode 100644 index 0000000000..bb8a80c496 --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml @@ -0,0 +1,3 @@ +name: test +dependencies: + - https://104.17.92.24:443::flask=1.0.2 diff --git a/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml-expected.json b/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml-expected.json new file mode 100644 index 0000000000..800bfcbab0 --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/test/environment_host_port.yml-expected.json @@ -0,0 +1,58 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "test", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": true, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:conda/flask@1.0.2", + "extracted_requirement": "=1.0.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_yaml", + "purl": "pkg:conda/test" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json b/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json index 499a73bf6a..ed97094e3f 100644 --- a/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json +++ b/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json @@ -33,7 +33,7 @@ "notice_text": null, "source_packages": [], "file_references": [], - "is_private": false, + "is_private": true, "is_virtual": false, "extra_data": { "channels": [ diff --git a/tests/packagedcode/test_conda.py b/tests/packagedcode/test_conda.py index 05df09149a..f6998485f1 100644 --- a/tests/packagedcode/test_conda.py +++ b/tests/packagedcode/test_conda.py @@ -113,3 +113,9 @@ def test_parse_conda_yaml_simple_dependencies(self): package = conda.CondaYamlHandler.parse(test_file) expected_loc = self.get_test_loc('conda/conda-yaml/ringer/environment.yaml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_yaml_does_not_fail_on_test_files_with_port(self): + test_file = self.get_test_loc('conda/conda-yaml/test/environment_host_port.yml') + package = conda.CondaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/conda-yaml/test/environment_host_port.yml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)