From 0da7e43353adfb871a726f36bca5d123c4d74a21 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 13:15:00 -0400 Subject: [PATCH 01/16] remove Provenance.provenance_digest The latest PEP 740 language doesn't include this, so we don't need it. Signed-off-by: William Woodruff --- requirements/main.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/main.txt b/requirements/main.txt index a02d2dcd29d0..0b4d54494192 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1988,7 +1988,6 @@ rfc8785==0.1.4 \ --hash=sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48 \ --hash=sha256:e545841329fe0eee4f6a3b44e7034343100c12b4ec566dc06ca9735681deb4da # via - # -r requirements/main.in # sigstore rich==13.9.1 \ --hash=sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466 \ From d0b1e7aee60c00ac18740587f92d25cba0e401ed Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 14:01:42 -0400 Subject: [PATCH 02/16] WIP route for provenance retrieval Signed-off-by: William Woodruff --- warehouse/attestations/views.py | 38 +++++++++++++++++++++++++++++++++ warehouse/packaging/models.py | 19 +++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 warehouse/attestations/views.py diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py new file mode 100644 index 000000000000..b73bd61c7f7c --- /dev/null +++ b/warehouse/attestations/views.py @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound +from pyramid.request import Request +from pyramid.view import view_config + +from warehouse.admin.flags import AdminFlagValue +from warehouse.packaging.models import File + + +@view_config( + route_name="attestations.provenance", + context=File, + require_methods=["GET"], + renderer="json", + require_csrf=False, + has_translations=False, +) +def provenance_for_release_file(file: File, request: Request): + if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): + return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) + + if not file.provenance: + return HTTPNotFound( + json={"message": f"No provenance available for {file.filename}"} + ) + + return file.provenance.provenance diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 2e4eb05fb2f7..3f10d47a0313 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -163,6 +163,25 @@ def __contains__(self, project): return True +class FileFactory: + def __init__(self, request): + self.request = request + + def __getitem__(self, filename): + try: + return self.request.db.query(File).filter(File.filename == filename).one() + except NoResultFound: + raise KeyError from None + + def __contains__(self, filename): + try: + self[filename] + except KeyError: + return False + else: + return True + + class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" From bb5d6b70778eb182b12944b05cf2e38d5fbed565 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 14:47:22 -0400 Subject: [PATCH 03/16] connect persistence, get tests passing Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index ed79477a52ec..28a7a33bf900 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -25,7 +25,7 @@ from tests.common.db.packaging import ProjectFactory, RoleFactory from warehouse.macaroons import caveats -from ...common.db.accounts import UserFactory +from ...common.db.accounts import EmailFactory, UserFactory from ...common.db.macaroons import MacaroonFactory _HERE = Path(__file__).parent @@ -329,3 +329,11 @@ def test_provenance_upload(webtest): assert len(attestations) == 1 attestation = attestations[0] assert attestation == json.loads(attestation_contents) + + # While we needed to be authenticated to upload a project, this is no longer + # required to view it. + webtest.authorization = None + expected_filename = "sampleproject-3.0.0.tar.gz" + + response = webtest.get(f"/_/provenance/{expected_filename}/", status=HTTPStatus.OK) + assert response.json == project.releases[0].files[0].provenance.provenance From 43f939f8d46694b6054539f820681eb5af0be719 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 23 Sep 2024 15:06:01 -0400 Subject: [PATCH 04/16] bring coverage up Signed-off-by: William Woodruff --- tests/unit/attestations/test_views.py | 36 ++++++++++++++++++++ tests/unit/packaging/test_models.py | 48 +++++++++++++++++++++++++++ warehouse/attestations/views.py | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/unit/attestations/test_views.py diff --git a/tests/unit/attestations/test_views.py b/tests/unit/attestations/test_views.py new file mode 100644 index 000000000000..ade93bff0fb7 --- /dev/null +++ b/tests/unit/attestations/test_views.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from warehouse.attestations import views + + +def test_provenance_for_file_not_enabled(): + request = pretend.stub( + flags=pretend.stub(enabled=lambda *a: True), + ) + + response = views.provenance_for_file(pretend.stub(), request) + assert response.status_code == 403 + assert response.json == {"message": "Attestations temporarily disabled"} + + +def test_provenance_for_file_not_present(): + request = pretend.stub( + flags=pretend.stub(enabled=lambda *a: False), + ) + file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") + + response = views.provenance_for_file(file, request) + assert response.status_code == 404 + assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index aac3cc37433b..b59420370706 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -25,6 +25,7 @@ from warehouse.organizations.models import TeamProjectRoleType from warehouse.packaging.models import ( File, + FileFactory, Project, ProjectFactory, ProjectMacaroonWarningAssociation, @@ -86,6 +87,53 @@ def test_contains(self, db_request): assert "bar" not in root +class TestFileFactory: + def test_traversal_finds(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + rfile_1 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.tar.gz", + python_version="source", + packagetype="sdist", + ) + rfile_2 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.whl", + python_version="bdist_wheel", + packagetype="bdist_wheel", + ) + + root = FileFactory(db_request) + assert root[rfile_1.filename] == rfile_1 + assert root[rfile_2.filename] == rfile_2 + + def test_travel_cant_find(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + + root = FileFactory(db_request) + + # Project and release exist, but no file exists. + with pytest.raises(KeyError): + root[f"{release.project.name}-{release.version}.tar.gz"] + + def test_contains(self, db_request): + project = DBProjectFactory.create(name="fakeproject") + release = DBReleaseFactory.create(project=project) + rfile_1 = DBFileFactory.create( + release=release, + filename=f"{release.project.name}-{release.version}.tar.gz", + python_version="source", + packagetype="sdist", + ) + + root = FileFactory(db_request) + + assert rfile_1.filename in root + assert (rfile_1.filename + ".invalid") not in root + + class TestProject: def test_traversal_finds(self, db_request): project = DBProjectFactory.create() diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py index b73bd61c7f7c..b75ef7e7fc4e 100644 --- a/warehouse/attestations/views.py +++ b/warehouse/attestations/views.py @@ -26,7 +26,7 @@ require_csrf=False, has_translations=False, ) -def provenance_for_release_file(file: File, request: Request): +def provenance_for_file(file: File, request: Request): if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) From f0afb8b657459f3a700e4bf426e79487364a4c2d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 24 Sep 2024 16:07:10 -0400 Subject: [PATCH 05/16] tests, warehouse: switch to a better route Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 5 ++- tests/unit/packaging/test_models.py | 48 ------------------------ warehouse/packaging/models.py | 19 ---------- 3 files changed, 4 insertions(+), 68 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index 28a7a33bf900..c7c73384f088 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -335,5 +335,8 @@ def test_provenance_upload(webtest): webtest.authorization = None expected_filename = "sampleproject-3.0.0.tar.gz" - response = webtest.get(f"/_/provenance/{expected_filename}/", status=HTTPStatus.OK) + response = webtest.get( + f"/metadata/{project.name}/3.0.0/{expected_filename}/provenance", + status=HTTPStatus.OK, + ) assert response.json == project.releases[0].files[0].provenance.provenance diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index b59420370706..aac3cc37433b 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -25,7 +25,6 @@ from warehouse.organizations.models import TeamProjectRoleType from warehouse.packaging.models import ( File, - FileFactory, Project, ProjectFactory, ProjectMacaroonWarningAssociation, @@ -87,53 +86,6 @@ def test_contains(self, db_request): assert "bar" not in root -class TestFileFactory: - def test_traversal_finds(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - rfile_1 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.tar.gz", - python_version="source", - packagetype="sdist", - ) - rfile_2 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.whl", - python_version="bdist_wheel", - packagetype="bdist_wheel", - ) - - root = FileFactory(db_request) - assert root[rfile_1.filename] == rfile_1 - assert root[rfile_2.filename] == rfile_2 - - def test_travel_cant_find(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - - root = FileFactory(db_request) - - # Project and release exist, but no file exists. - with pytest.raises(KeyError): - root[f"{release.project.name}-{release.version}.tar.gz"] - - def test_contains(self, db_request): - project = DBProjectFactory.create(name="fakeproject") - release = DBReleaseFactory.create(project=project) - rfile_1 = DBFileFactory.create( - release=release, - filename=f"{release.project.name}-{release.version}.tar.gz", - python_version="source", - packagetype="sdist", - ) - - root = FileFactory(db_request) - - assert rfile_1.filename in root - assert (rfile_1.filename + ".invalid") not in root - - class TestProject: def test_traversal_finds(self, db_request): project = DBProjectFactory.create() diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 3f10d47a0313..2e4eb05fb2f7 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -163,25 +163,6 @@ def __contains__(self, project): return True -class FileFactory: - def __init__(self, request): - self.request = request - - def __getitem__(self, filename): - try: - return self.request.db.query(File).filter(File.filename == filename).one() - except NoResultFound: - raise KeyError from None - - def __contains__(self, filename): - try: - self[filename] - except KeyError: - return False - else: - return True - - class LifecycleStatus(enum.StrEnum): QuarantineEnter = "quarantine-enter" QuarantineExit = "quarantine-exit" From c333e65386dc21a93b044d7a9ae42d25138cf342 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 26 Sep 2024 13:39:39 -0400 Subject: [PATCH 06/16] rename route, content negotiation Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 2 +- tests/unit/attestations/test_views.py | 35 +++++++++++++++------- warehouse/attestations/views.py | 37 +++++++++++++++++++++++- 3 files changed, 62 insertions(+), 12 deletions(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index c7c73384f088..5dfcf8377b56 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -336,7 +336,7 @@ def test_provenance_upload(webtest): expected_filename = "sampleproject-3.0.0.tar.gz" response = webtest.get( - f"/metadata/{project.name}/3.0.0/{expected_filename}/provenance", + f"/integrity/{project.name}/3.0.0/{expected_filename}/provenance", status=HTTPStatus.OK, ) assert response.json == project.releases[0].files[0].provenance.provenance diff --git a/tests/unit/attestations/test_views.py b/tests/unit/attestations/test_views.py index ade93bff0fb7..172833820b99 100644 --- a/tests/unit/attestations/test_views.py +++ b/tests/unit/attestations/test_views.py @@ -11,26 +11,41 @@ # limitations under the License. import pretend +import pytest from warehouse.attestations import views -def test_provenance_for_file_not_enabled(): - request = pretend.stub( - flags=pretend.stub(enabled=lambda *a: True), - ) +def test_select_content_type(db_request): + db_request.accept = "application/json" - response = views.provenance_for_file(pretend.stub(), request) + assert views._select_content_type(db_request) == views.MIME_PYPI_INTEGRITY_V1_JSON + + +# Backstop; can be removed/changed once this view supports HTML. +@pytest.mark.parametrize( + "content_type", + [views.MIME_TEXT_HTML, views.MIME_PYPI_INTEGRITY_V1_HTML], +) +def test_provenance_for_file_bad_accept(db_request, content_type): + db_request.accept = content_type + response = views.provenance_for_file(pretend.stub(), db_request) + assert response.status_code == 406 + assert response.json == {"message": "Request not acceptable"} + + +def test_provenance_for_file_not_enabled(db_request, monkeypatch): + monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True)) + + response = views.provenance_for_file(pretend.stub(), db_request) assert response.status_code == 403 assert response.json == {"message": "Attestations temporarily disabled"} -def test_provenance_for_file_not_present(): - request = pretend.stub( - flags=pretend.stub(enabled=lambda *a: False), - ) +def test_provenance_for_file_not_present(db_request, monkeypatch): + monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False)) file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") - response = views.provenance_for_file(file, request) + response = views.provenance_for_file(file, db_request) assert response.status_code == 404 assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py index b75ef7e7fc4e..dc6d70404da7 100644 --- a/warehouse/attestations/views.py +++ b/warehouse/attestations/views.py @@ -10,12 +10,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound +from pyramid.httpexceptions import HTTPForbidden, HTTPNotAcceptable, HTTPNotFound from pyramid.request import Request from pyramid.view import view_config from warehouse.admin.flags import AdminFlagValue +from warehouse.cache.http import add_vary from warehouse.packaging.models import File +from warehouse.utils.cors import _CORS_HEADERS + +MIME_TEXT_HTML = "text/html" +MIME_PYPI_INTEGRITY_V1_HTML = "application/vnd.pypi.integrity.v1+html" +MIME_PYPI_INTEGRITY_V1_JSON = "application/vnd.pypi.integrity.v1+json" + + +def _select_content_type(request: Request) -> str: + offers = request.accept.acceptable_offers( + [ + # JSON currently has the highest priority. + MIME_PYPI_INTEGRITY_V1_JSON, + MIME_TEXT_HTML, + MIME_PYPI_INTEGRITY_V1_HTML, + ] + ) + + # Default case: JSON. + if not offers: + return MIME_PYPI_INTEGRITY_V1_JSON + else: + return offers[0][0] @view_config( @@ -25,8 +48,17 @@ renderer="json", require_csrf=False, has_translations=False, + decorator=[ + add_vary("Accept"), + ], ) def provenance_for_file(file: File, request: Request): + # Determine our response content-type. For the time being, only the JSON + # type is accepted. + request.response.content_type = _select_content_type(request) + if request.response.content_type != MIME_PYPI_INTEGRITY_V1_JSON: + return HTTPNotAcceptable(json={"message": "Request not acceptable"}) + if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) @@ -35,4 +67,7 @@ def provenance_for_file(file: File, request: Request): json={"message": f"No provenance available for {file.filename}"} ) + # Apply CORS headers. + request.response.headers.update(_CORS_HEADERS) + return file.provenance.provenance From 13f525849e310ccea04c072aa55f35a340d2528f Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 26 Sep 2024 13:57:12 -0400 Subject: [PATCH 07/16] attestations.views -> api.integrity Signed-off-by: William Woodruff --- tests/unit/attestations/test_views.py | 51 ------------------- warehouse/attestations/views.py | 73 --------------------------- 2 files changed, 124 deletions(-) delete mode 100644 tests/unit/attestations/test_views.py delete mode 100644 warehouse/attestations/views.py diff --git a/tests/unit/attestations/test_views.py b/tests/unit/attestations/test_views.py deleted file mode 100644 index 172833820b99..000000000000 --- a/tests/unit/attestations/test_views.py +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pretend -import pytest - -from warehouse.attestations import views - - -def test_select_content_type(db_request): - db_request.accept = "application/json" - - assert views._select_content_type(db_request) == views.MIME_PYPI_INTEGRITY_V1_JSON - - -# Backstop; can be removed/changed once this view supports HTML. -@pytest.mark.parametrize( - "content_type", - [views.MIME_TEXT_HTML, views.MIME_PYPI_INTEGRITY_V1_HTML], -) -def test_provenance_for_file_bad_accept(db_request, content_type): - db_request.accept = content_type - response = views.provenance_for_file(pretend.stub(), db_request) - assert response.status_code == 406 - assert response.json == {"message": "Request not acceptable"} - - -def test_provenance_for_file_not_enabled(db_request, monkeypatch): - monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True)) - - response = views.provenance_for_file(pretend.stub(), db_request) - assert response.status_code == 403 - assert response.json == {"message": "Attestations temporarily disabled"} - - -def test_provenance_for_file_not_present(db_request, monkeypatch): - monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False)) - file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz") - - response = views.provenance_for_file(file, db_request) - assert response.status_code == 404 - assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"} diff --git a/warehouse/attestations/views.py b/warehouse/attestations/views.py deleted file mode 100644 index dc6d70404da7..000000000000 --- a/warehouse/attestations/views.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pyramid.httpexceptions import HTTPForbidden, HTTPNotAcceptable, HTTPNotFound -from pyramid.request import Request -from pyramid.view import view_config - -from warehouse.admin.flags import AdminFlagValue -from warehouse.cache.http import add_vary -from warehouse.packaging.models import File -from warehouse.utils.cors import _CORS_HEADERS - -MIME_TEXT_HTML = "text/html" -MIME_PYPI_INTEGRITY_V1_HTML = "application/vnd.pypi.integrity.v1+html" -MIME_PYPI_INTEGRITY_V1_JSON = "application/vnd.pypi.integrity.v1+json" - - -def _select_content_type(request: Request) -> str: - offers = request.accept.acceptable_offers( - [ - # JSON currently has the highest priority. - MIME_PYPI_INTEGRITY_V1_JSON, - MIME_TEXT_HTML, - MIME_PYPI_INTEGRITY_V1_HTML, - ] - ) - - # Default case: JSON. - if not offers: - return MIME_PYPI_INTEGRITY_V1_JSON - else: - return offers[0][0] - - -@view_config( - route_name="attestations.provenance", - context=File, - require_methods=["GET"], - renderer="json", - require_csrf=False, - has_translations=False, - decorator=[ - add_vary("Accept"), - ], -) -def provenance_for_file(file: File, request: Request): - # Determine our response content-type. For the time being, only the JSON - # type is accepted. - request.response.content_type = _select_content_type(request) - if request.response.content_type != MIME_PYPI_INTEGRITY_V1_JSON: - return HTTPNotAcceptable(json={"message": "Request not acceptable"}) - - if request.flags.enabled(AdminFlagValue.DISABLE_PEP740): - return HTTPForbidden(json={"message": "Attestations temporarily disabled"}) - - if not file.provenance: - return HTTPNotFound( - json={"message": f"No provenance available for {file.filename}"} - ) - - # Apply CORS headers. - request.response.headers.update(_CORS_HEADERS) - - return file.provenance.provenance From f5603506aa31915ddfe6bf4559f9372b84fa185d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 27 Sep 2024 13:50:27 -0400 Subject: [PATCH 08/16] Add provenance to simple API Broken out from #16624. Signed-off-by: William Woodruff --- tests/unit/api/test_simple.py | 3 +++ warehouse/packaging/utils.py | 12 +++++++++++- warehouse/templates/api/simple/detail.html | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 518a3dea2b95..dde4f2af87b5 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -298,6 +298,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override) "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -349,6 +350,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid "upload-time": f.upload_time.isoformat() + "Z", "data-dist-info-metadata": False, "core-metadata": False, + "provenance": None, } for f in files ], @@ -445,6 +447,7 @@ def test_with_files_with_version_multi_digit( if f.metadata_file_sha256_digest is not None else False ), + "provenance": None, } for f in files ], diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 282c45747f75..f31738d081c0 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -22,7 +22,7 @@ from warehouse.packaging.interfaces import ISimpleStorage from warehouse.packaging.models import File, LifecycleStatus, Project, Release -API_VERSION = "1.2" +API_VERSION = "1.3" def _simple_index(request, serial): @@ -101,6 +101,16 @@ def _simple_detail(project, request): if file.metadata_file_sha256_digest else False ), + "provenance": ( + request.route_url( + "attestations.provenance", + project_name=project.normalized_name, + release=file.release.version, + filename=file.filename, + ) + if file.provenance + else None + ), } for file in files ], diff --git a/warehouse/templates/api/simple/detail.html b/warehouse/templates/api/simple/detail.html index 28a8f49c815d..77a73dc63fb6 100644 --- a/warehouse/templates/api/simple/detail.html +++ b/warehouse/templates/api/simple/detail.html @@ -23,7 +23,7 @@

Links for {{ name }}

{% for file in files -%} - {{ file.filename }}
+ {{ file.filename }}
{% endfor -%} From 47c57ddbb7ccbfc62b8a25ef54d38002e790f2ad Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 27 Sep 2024 14:13:16 -0400 Subject: [PATCH 09/16] api: unit level test for provenance in simple API Signed-off-by: William Woodruff --- tests/unit/api/test_simple.py | 98 +++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index dde4f2af87b5..3c5c5f34f909 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -17,6 +17,7 @@ from pyramid.httpexceptions import HTTPMovedPermanently from pyramid.testing import DummyRequest +from tests.common.db.oidc import GitHubPublisherFactory from warehouse.api import simple from warehouse.packaging.utils import API_VERSION, _valid_simple_detail_context @@ -26,6 +27,7 @@ FileFactory, JournalEntryFactory, ProjectFactory, + ProvenanceFactory, ReleaseFactory, ) @@ -492,6 +494,102 @@ def test_with_files_quarantined_omitted_from_index( if renderer_override is not None: assert db_request.override_renderer == renderer_override + @pytest.mark.parametrize( + ("content_type", "renderer_override"), + CONTENT_TYPE_PARAMS, + ) + def test_with_files_varying_provenance( + self, + db_request, + integrity_service, + dummy_attestation, + content_type, + renderer_override, + ): + db_request.accept = content_type + db_request.oidc_publisher = GitHubPublisherFactory.create() + + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0.0") + + # wheel with provenance, sdist with no provenance + wheel = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.whl", + packagetype="bdist_wheel", + metadata_file_sha256_digest="deadbeefdeadbeefdeadbeefdeadbeef", + ) + + provenance = ProvenanceFactory.create(file=wheel) + assert wheel.provenance is not None + assert wheel.provenance == provenance + + sdist = FileFactory.create( + release=release, + filename=f"{project.name}-1.0.0.tar.gz", + packagetype="sdist", + ) + + files = [sdist, wheel] + + urls_iter = (f"/file/{f.filename}" for f in files) + provenance_iter = ( + ( + f"/integrity/{f.release.project.normalized_name}/" + f"{f.release.version}/{f.filename}/provenance" + ) + for f in [wheel] + ) + db_request.matchdict["name"] = project.normalized_name + db_request.route_url = lambda r, **kw: ( + next(urls_iter) if r == "packaging.file" else next(provenance_iter) + ) + user = UserFactory.create() + je = JournalEntryFactory.create(name=project.name, submitted_by=user) + + context = { + "meta": {"_last-serial": je.id, "api-version": API_VERSION}, + "name": project.normalized_name, + "versions": ["1.0.0"], + "files": [ + { + "filename": f.filename, + "url": f"/file/{f.filename}", + "hashes": {"sha256": f.sha256_digest}, + "requires-python": f.requires_python, + "yanked": False, + "size": f.size, + "upload-time": f.upload_time.isoformat() + "Z", + "data-dist-info-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "core-metadata": ( + {"sha256": "deadbeefdeadbeefdeadbeefdeadbeef"} + if f.metadata_file_sha256_digest is not None + else False + ), + "provenance": ( + ( + f"/integrity/{f.release.project.normalized_name}/" + f"{f.release.version}/{f.filename}/provenance" + ) + if f.provenance is not None + else None + ), + } + for f in files + ], + "alternate-locations": [], + } + context = _update_context(context, content_type, renderer_override) + + assert simple.simple_detail(project, db_request) == context + + if renderer_override is not None: + assert db_request.override_renderer == renderer_override + def _update_context(context, content_type, renderer_override): if renderer_override != "json" or content_type in [ From ba557ee8ab77d4671f2c1cfd37a3f44487e2094d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 27 Sep 2024 14:24:13 -0400 Subject: [PATCH 10/16] api: functional test for provenance in simple API Signed-off-by: William Woodruff --- tests/functional/api/test_simple.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 7d97f8dc284a..02e96387f5cb 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -12,7 +12,12 @@ from http import HTTPStatus -from ...common.db.packaging import FileFactory, ProjectFactory, ReleaseFactory +from ...common.db.packaging import ( + FileFactory, + ProjectFactory, + ProvenanceFactory, + ReleaseFactory, +) def test_simple_api_html(webtest): @@ -34,3 +39,24 @@ def test_simple_api_detail(webtest): assert resp.html.h1.string == f"Links for {project.normalized_name}" # There should be a link for every file assert len(resp.html.find_all("a")) == 2 + + +def test_simple_api_has_provenance(webtest): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project) + files = FileFactory.create_batch(2, release=release, packagetype="bdist_wheel") + + for file in files: + ProvenanceFactory.create(file=file) + + resp = webtest.get(f"/simple/{project.normalized_name}/", status=HTTPStatus.OK) + links = resp.html.find_all("a") + + for link, file in zip(links, files): + provenance_url = link.get("data-provenance") + assert provenance_url is not None + + assert provenance_url == ( + f"http://localhost/integrity/{file.release.project.normalized_name}/" + f"{file.release.version}/{file.filename}/provenance" + ) From 7993e2d3559683113616da8eec1fac8a5476d7ab Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Oct 2024 12:17:37 -0400 Subject: [PATCH 11/16] fixup tests Signed-off-by: William Woodruff --- tests/functional/api/test_simple.py | 3 ++- warehouse/packaging/utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 02e96387f5cb..7d2169413105 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -52,7 +52,8 @@ def test_simple_api_has_provenance(webtest): resp = webtest.get(f"/simple/{project.normalized_name}/", status=HTTPStatus.OK) links = resp.html.find_all("a") - for link, file in zip(links, files): + for file in files: + link = next(link for link in links if link.text == file.filename) provenance_url = link.get("data-provenance") assert provenance_url is not None diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index f31738d081c0..7397cf45a740 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -103,7 +103,7 @@ def _simple_detail(project, request): ), "provenance": ( request.route_url( - "attestations.provenance", + "integrity.provenance", project_name=project.normalized_name, release=file.release.version, filename=file.filename, From 1d93b0cf10ab9c8a866a440f7bf4107fa7fddf7c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 3 Oct 2024 12:28:08 -0400 Subject: [PATCH 12/16] unused import Signed-off-by: William Woodruff --- tests/functional/forklift/test_legacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index 5dfcf8377b56..3f0bc7dbd684 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -25,7 +25,7 @@ from tests.common.db.packaging import ProjectFactory, RoleFactory from warehouse.macaroons import caveats -from ...common.db.accounts import EmailFactory, UserFactory +from ...common.db.accounts import UserFactory from ...common.db.macaroons import MacaroonFactory _HERE = Path(__file__).parent From 904c69140280dac11aca8a18737080a4850f976c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 16 Oct 2024 22:05:12 +0100 Subject: [PATCH 13/16] Apply suggestions from code review Co-authored-by: Mike Fiedler --- tests/functional/api/test_simple.py | 1 - tests/unit/api/test_simple.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/functional/api/test_simple.py b/tests/functional/api/test_simple.py index 7d2169413105..909037bc1699 100644 --- a/tests/functional/api/test_simple.py +++ b/tests/functional/api/test_simple.py @@ -55,7 +55,6 @@ def test_simple_api_has_provenance(webtest): for file in files: link = next(link for link in links if link.text == file.filename) provenance_url = link.get("data-provenance") - assert provenance_url is not None assert provenance_url == ( f"http://localhost/integrity/{file.release.project.normalized_name}/" diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 3c5c5f34f909..81b363710ae4 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -521,7 +521,6 @@ def test_with_files_varying_provenance( ) provenance = ProvenanceFactory.create(file=wheel) - assert wheel.provenance is not None assert wheel.provenance == provenance sdist = FileFactory.create( From ba9cddde07eaec800a22c48d9af394a34341758d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 16 Oct 2024 22:21:00 +0100 Subject: [PATCH 14/16] test_simple: clean up route_url Signed-off-by: William Woodruff --- tests/unit/api/test_simple.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 81b363710ae4..5fa2b4e191ed 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -531,6 +531,13 @@ def test_with_files_varying_provenance( files = [sdist, wheel] + db_request.matchdict["name"] = project.normalized_name + + # Rendering the simple index calls route_url once for each file, + # and zero or one times per file depending on whether the file + # has provenance. We emulate this while testing by maintaining + # two iters of URLs we want to return, each of which will be + # pulled from as appropriate when a route_url call is made. urls_iter = (f"/file/{f.filename}" for f in files) provenance_iter = ( ( @@ -539,10 +546,17 @@ def test_with_files_varying_provenance( ) for f in [wheel] ) - db_request.matchdict["name"] = project.normalized_name - db_request.route_url = lambda r, **kw: ( - next(urls_iter) if r == "packaging.file" else next(provenance_iter) - ) + + def route_url(route, **_kw): + if route == "packaging.file": + return next(urls_iter) + elif route == "integrity.provenance": + return next(provenance_iter) + else: + assert False, route + + db_request.route_url = route_url + user = UserFactory.create() je = JournalEntryFactory.create(name=project.name, submitted_by=user) From 584922f077ce4b9783e444aeaddbe985bd4d0e3c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 16 Oct 2024 22:40:51 +0100 Subject: [PATCH 15/16] fix lint Signed-off-by: William Woodruff --- tests/unit/api/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index 5fa2b4e191ed..fac2eb4380a4 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -553,7 +553,7 @@ def route_url(route, **_kw): elif route == "integrity.provenance": return next(provenance_iter) else: - assert False, route + pytest.fail(f"unexpected route: {route}") db_request.route_url = route_url From bcbdc7f3fd84510e6428af805fc538872e0d3b7e Mon Sep 17 00:00:00 2001 From: Mike Fiedler Date: Fri, 18 Oct 2024 16:19:39 -0400 Subject: [PATCH 16/16] test: refactor test case to provide clearer routes Signed-off-by: Mike Fiedler --- tests/unit/api/test_simple.py | 53 +++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/tests/unit/api/test_simple.py b/tests/unit/api/test_simple.py index fac2eb4380a4..ce05298afd91 100644 --- a/tests/unit/api/test_simple.py +++ b/tests/unit/api/test_simple.py @@ -533,27 +533,38 @@ def test_with_files_varying_provenance( db_request.matchdict["name"] = project.normalized_name - # Rendering the simple index calls route_url once for each file, - # and zero or one times per file depending on whether the file - # has provenance. We emulate this while testing by maintaining - # two iters of URLs we want to return, each of which will be - # pulled from as appropriate when a route_url call is made. - urls_iter = (f"/file/{f.filename}" for f in files) - provenance_iter = ( - ( - f"/integrity/{f.release.project.normalized_name}/" - f"{f.release.version}/{f.filename}/provenance" - ) - for f in [wheel] - ) - - def route_url(route, **_kw): - if route == "packaging.file": - return next(urls_iter) - elif route == "integrity.provenance": - return next(provenance_iter) - else: - pytest.fail(f"unexpected route: {route}") + def route_url(route, **kw): + # Rendering the simple index calls route_url once for each file, + # and zero or one times per file depending on whether the file + # has provenance. We emulate this while testing by maintaining + # a dictionary of expected URLs for each route, which are + # pulled from as appropriate when a route_url call is made. + route_urls = { + "packaging.file": {f.path: f"/file/{f.filename}" for f in files}, + "integrity.provenance": { + ( + wheel.release.project.normalized_name, + wheel.release.version, + wheel.filename, + ): ( + f"/integrity/{wheel.release.project.normalized_name}/" + f"{wheel.release.version}/{wheel.filename}/provenance" + ) + }, + } + + match route: + case "packaging.file": + return route_urls[route].get(kw.get("path"), "") + case "integrity.provenance": + key = ( + kw.get("project_name"), + kw.get("release"), + kw.get("filename"), + ) + return route_urls[route].get(key, "") + case _: + pytest.fail(f"unexpected route: {route}") db_request.route_url = route_url