Store attestations for PEP740 (#16302)

* Store and retrieve attestations

* Continue to work.

* Update attestation storage and retrieval

* Update comments

* Fix import

* Please linter

* Use the correct event to attach a publisher_url to a `File`

* Rename ReleaseFileAttestation to Attestation

* Update names

* Update table migration

* Update metrics

* Update attestations storage

* Update pypi-attestations and sigstore dependencies

* Fix wrong merge.

* Generate Provenance file on upload.

* Generate and store provenance file during upload.

* Improve tests

* Fix test error

* Fix test error

* Fix merge error

* Simplify legacy answer

* Introduce AttestationsService

* Rename AttestationsService to ReleaseVerification service and integrate Provenance related objects

* Remove useless check

* Integrate generate_and_store_provenance within persist_attestations

* Remove file.publisher_url which is no longer used.

* Rename ReleaseAttestationService to IntegrityService

* Linting

* requirements: bump sigstore, pypi-attestations

Signed-off-by: William Woodruff <william@trailofbits.com>

---------

Signed-off-by: William Woodruff <william@trailofbits.com>
Co-authored-by: Dustin Ingram <di@users.noreply.github.com>
Co-authored-by: William Woodruff <william@trailofbits.com>

requirements/main.in

@@ -62,8 +62,8 @@ redis>=2.8.0,<6.0.0
 rfc3986
 sentry-sdk
 setuptools
-sigstore~=3.0.0
-pypi-attestations==0.0.9
+sigstore~=3.2.0
+pypi-attestations==0.0.11
 sqlalchemy[asyncio]>=2.0,<3.0
 stdlib-list
 stripe

requirements/main.txt

@@ -1770,9 +1770,9 @@ pyparsing==3.1.2 \
     --hash=sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad \
     --hash=sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742
     # via linehaul
-pypi-attestations==0.0.9 \
-    --hash=sha256:3bfc07f64a8db0d6e2646720e70df7c7cb01a2936056c764a2cc3268969332f2 \
-    --hash=sha256:4b38cce5d221c8145cac255bfafe650ec0028d924d2b3572394df8ba8f07a609
+pypi-attestations==0.0.11 \
+    --hash=sha256:b730e6b23874d94da0f3817b1f9dd3ecb6a80d685f62a18ad96e5b0396149ded \
+    --hash=sha256:e74329074f049568591e300373e12fcd46a35e21723110856546e33bf2949efa
     # via -r requirements/main.in
 pyqrcode==1.2.1 \
     --hash=sha256:1b2812775fa6ff5c527977c4cd2ccb07051ca7d0bc0aecf937a43864abe5eff6 \
@@ -2079,9 +2079,9 @@ sentry-sdk==2.13.0 \
     --hash=sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6 \
     --hash=sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260
     # via -r requirements/main.in
-sigstore==3.0.0 \
-    --hash=sha256:6cc7dc92607c2fd481aada0f3c79e710e4c6086e3beab50b07daa9a50a79d109 \
-    --hash=sha256:a6a9538a648e112a0c3d8092d3f73a351c7598164764f1e73a6b5ba406a3a0bd
+sigstore==3.2.0 \
+    --hash=sha256:25c8a871a3a6adf959c0cde598ea8bef8794f1a29277d067111eb4ded4ba7f65 \
+    --hash=sha256:d18508f34febb7775065855e92557fa1c2c16580df88f8e8903b9514438bad44
     # via
     #   -r requirements/main.in
     #   pypi-attestations

tests/common/db/attestation.py

@@ -0,0 +1,28 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+
+import factory
+
+from warehouse.attestations.models import Attestation
+
+from .base import WarehouseFactory
+
+
+class AttestationFactory(WarehouseFactory):
+    class Meta:
+        model = Attestation
+
+    file = factory.SubFactory("tests.common.db.packaging.FileFactory")
+    attestation_file_blake2_digest = factory.LazyAttribute(
+        lambda o: hashlib.blake2b(o.file.filename.encode("utf8")).hexdigest()
+    )

tests/common/db/packaging.py

@@ -34,6 +34,7 @@
 from warehouse.utils import readme
 
 from .accounts import UserFactory
+from .attestation import AttestationFactory
 from .base import WarehouseFactory
 from .observations import ObserverFactory
 
@@ -140,6 +141,12 @@ class Meta:
         )
     )
 
+    attestations = factory.RelatedFactoryList(
+        AttestationFactory,
+        factory_related_name="file",
+        size=1,
+    )
+
 
 class FileEventFactory(WarehouseFactory):
     class Meta:

tests/unit/api/test_simple.py

@@ -18,6 +18,7 @@
 from pyramid.testing import DummyRequest
 
 from warehouse.api import simple
+from warehouse.attestations import IIntegrityService
 from warehouse.packaging.utils import API_VERSION
 
 from ...common.db.accounts import UserFactory
@@ -87,6 +88,16 @@ def test_selects(self, header, expected):
 
 
 class TestSimpleIndex:
+
+    @pytest.fixture
+    def db_request(self, db_request):
+        """Override db_request to add the Release Verification service"""
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IIntegrityService: pretend.stub(),
+        }.get(svc)
+
+        return db_request
+
     @pytest.mark.parametrize(
         ("content_type", "renderer_override"),
         CONTENT_TYPE_PARAMS,
@@ -185,6 +196,17 @@ def test_quarantined_project_omitted_from_index(self, db_request):
 
 
 class TestSimpleDetail:
+    @pytest.fixture
+    def db_request(self, db_request):
+        """Override db_request to add the Release Verification service"""
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IIntegrityService: pretend.stub(
+                get_provenance_digest=lambda *args, **kwargs: None,
+            ),
+        }.get(svc)
+
+        return db_request
+
     def test_redirects(self, pyramid_request):
         project = pretend.stub(normalized_name="foo")
 
@@ -286,6 +308,7 @@ def test_with_files_no_serial(self, db_request, content_type, renderer_override)
                     "upload-time": f.upload_time.isoformat() + "Z",
                     "data-dist-info-metadata": False,
                     "core-metadata": False,
+                    "provenance": None,
                 }
                 for f in files
             ],
@@ -334,6 +357,7 @@ def test_with_files_with_serial(self, db_request, content_type, renderer_overrid
                     "upload-time": f.upload_time.isoformat() + "Z",
                     "data-dist-info-metadata": False,
                     "core-metadata": False,
+                    "provenance": None,
                 }
                 for f in files
             ],
@@ -427,6 +451,7 @@ def test_with_files_with_version_multi_digit(
                         if f.metadata_file_sha256_digest is not None
                         else False
                     ),
+                    "provenance": None,
                 }
                 for f in files
             ],

tests/unit/attestations/__init__.py

@@ -0,0 +1,11 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.