Manage packages/project and simple details to TUF

* Adding packages

After adding a package to the Warehouse database, it generates and
stores the Simple Index with a request to the RSTUF backend to
include the package and its simple index in TUF Metadata.

* Removing package or Project Release

On PyPI Management, when a user removes a file or a project release
it also removes it from TUF metadata and updates the simple details index.

Co-authored-by: Lukas Puehringer <lukas.puehringer@nyu.edu>
Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>

simplify code in warehouse.tuf.targets

Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>

Author: Kairo de Araujo

.vscode/settings.json

@@ -69,6 +69,7 @@
 )
 from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files
 from warehouse.rate_limiting.interfaces import RateLimiterException
+from warehouse.tuf import targets
 from warehouse.utils import http, readme
 from warehouse.utils.project import PROJECT_NAME_RE, validate_project_name
 from warehouse.utils.security_policy import AuthenticationMethod
@@ -1405,6 +1406,9 @@ def file_upload(request):
         file_data = file_
         request.db.add(file_)
 
+        # Add the project simple detail and file to TUF Metadata
+        task = targets.add_file(request, project, file_)
+
         file_.record_event(
             tag=EventTag.File.FileAdd,
             request=request,
@@ -1420,6 +1424,7 @@ def file_upload(request):
                 if request.oidc_publisher
                 else None,
                 "project_id": str(project.id),
+                "tuf": task["data"]["task_id"],
             },
         )
 

warehouse/forklift/legacy.py

@@ -122,6 +122,7 @@
     RoleInvitationStatus,
 )
 from warehouse.rate_limiting import IRateLimiter
+from warehouse.tuf import targets
 from warehouse.utils.http import is_safe_url
 from warehouse.utils.paginate import paginate_url_factory
 from warehouse.utils.project import confirm_project, destroy_docs, remove_project
@@ -1835,17 +1836,24 @@ def delete_project_release(self):
             )
         )
 
+        # Delete the project release (simple detail and files) from TUF Metadata
+        tasks = targets.delete_release(self.request, self.release)
+
         self.release.project.record_event(
             tag=EventTag.Project.ReleaseRemove,
             request=self.request,
             additional={
                 "submitted_by": self.request.user.username,
                 "canonical_version": self.release.canonical_version,
+                "tuf": ", ".join([task["data"]["task_id"] for task in tasks]),
             },
         )
 
         self.request.db.delete(self.release)
 
+        # Generate new project simple detail and add to TUF Metadata
+        targets.add_file(self.request, self.release.project)
+
         self.request.session.flash(
             self.request._(f"Deleted release {self.release.version!r}"), queue="success"
         )
@@ -1927,6 +1935,9 @@ def _error(message):
             )
         )
 
+        # Delete the file and project simple detail from TUF metadata
+        task = targets.delete_file(self.request, self.release.project, release_file)
+
         release_file.record_event(
             tag=EventTag.File.FileRemove,
             request=self.request,
@@ -1935,6 +1946,7 @@ def _error(message):
                 "canonical_version": self.release.canonical_version,
                 "filename": release_file.filename,
                 "project_id": str(self.release.project.id),
+                "tuf": task["data"]["task_id"],
             },
         )
 
@@ -1959,6 +1971,9 @@ def _error(message):
 
         self.request.db.delete(release_file)
 
+        # Generate new project simple detail and add to TUF Metadata
+        targets.add_file(self.request, self.release.project)
+
         self.request.session.flash(
             f"Deleted file {release_file.filename!r}", queue="success"
         )

warehouse/manage/views/__init__.py

@@ -44,6 +44,11 @@ def get_checksum(path):
         Return the md5 digest of the file at a given path as a lowercase string.
         """
 
+    def get_blake2bsum(path):
+        """
+        Return the blake2b digest of the file at a given path as a lowercase string.
+        """
+
     def store(path, file_path, *, meta=None):
         """
         Save the file located at file_path to the file storage at the location

warehouse/packaging/interfaces.py

@@ -74,6 +74,13 @@ def get_metadata(self, path):
     def get_checksum(self, path):
         return hashlib.md5(open(os.path.join(self.base, path), "rb").read()).hexdigest()
 
+    def get_blake2bsum(self, path):
+        content_hasher = hashlib.blake2b(digest_size=256 // 8)
+        content_hasher.update(open(os.path.join(self.base, path), "rb").read())
+        content_hash = content_hasher.hexdigest().lower()
+
+        return content_hash
+
     def store(self, path, file_path, *, meta=None):
         destination = os.path.join(self.base, path)
         os.makedirs(os.path.dirname(destination), exist_ok=True)
@@ -195,6 +202,9 @@ def create_service(cls, context, request):
         prefix = request.registry.settings.get("files.prefix")
         return cls(bucket, prefix=prefix)
 
+    def get_blake2bsum(self, path):
+        raise NotImplementedError
+
 
 class GenericS3BlobStorage(GenericBlobStorage):
     def get(self, path):
@@ -247,6 +257,9 @@ def create_service(cls, context, request):
         prefix = request.registry.settings.get("files.prefix")
         return cls(bucket, prefix=prefix)
 
+    def get_blake2bsum(self, path):
+        raise NotImplementedError
+
 
 @implementer(IFileStorage)
 class S3ArchiveFileStorage(GenericS3BlobStorage):
@@ -258,6 +271,9 @@ def create_service(cls, context, request):
         prefix = request.registry.settings.get("archive_files.prefix")
         return cls(bucket, prefix=prefix)
 
+    def get_blake2bsum(self, path):
+        raise NotImplementedError
+
 
 @implementer(IDocsStorage)
 class S3DocsStorage:
@@ -305,6 +321,20 @@ def get_metadata(self, path):
     def get_checksum(self, path):
         raise NotImplementedError
 
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def get_blake2bsum(self, path):
+        path = self._get_path(path)
+        blob = self.bucket.blob(path)
+        content_hasher = hashlib.blake2b(digest_size=256 // 8)
+        content_hasher.update(blob.download_as_string())
+        content_hash = content_hasher.hexdigest().lower()
+
+        return content_hash
+
     @google.api_core.retry.Retry(
         predicate=google.api_core.retry.if_exception_type(
             google.api_core.exceptions.ServiceUnavailable

warehouse/packaging/services.py

@@ -121,3 +121,13 @@ def render_simple_detail(project, request, store=False):
             )
 
     return (content_hash, simple_detail_path, simple_detail_size)
+
+
+def current_simple_details_path(request, project):
+    storage = request.find_service(ISimpleStorage)
+    current_hash = storage.get_blake2bsum(f"{project.normalized_name}/index.html")
+    simple_detail_path = (
+        f"{project.normalized_name}/{current_hash}.{project.normalized_name}.html"
+    )
+
+    return simple_detail_path

warehouse/packaging/utils.py

@@ -14,4 +14,5 @@
 def includeme(config):
     api_base_url = config.registry.settings["tuf.api.url"]
     config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"})
+    config.add_settings({"tuf.api.targets.url": f"{api_base_url}targets/"})
     config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"})

warehouse/tuf/__init__.py

@@ -0,0 +1,91 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+
+from pyramid.httpexceptions import HTTPBadGateway
+
+from warehouse.packaging.models import File
+from warehouse.packaging.utils import current_simple_details_path, render_simple_detail
+
+
+def _payload(targets):
+    """Helper to create payload for POST or DELETE targets request."""
+    return {
+        "targets": targets,
+        "publish_targets": True,
+    }
+
+
+def _payload_targets_part(path, size, digest):
+    """Helper to create payload part for POST targets request."""
+    return {
+        "path": path,
+        "info": {
+            "length": size,
+            "hashes": {"blake2b-256": digest},
+        },
+    }
+
+
+def _handle(response):
+    """Helper to handle http response for POST or DELETE targets request."""
+    if response.status_code != 202:
+        raise HTTPBadGateway(f"Unexpected TUF Server response: {response.text}")
+
+    return response.json()
+
+
+def add_file(request, project, file=None):
+    """Call RSTUF to add file and new project simple index to TUF targets metadata.
+
+    NOTE: If called without file, only adds new project simple index. This
+    can be used to re-add project simple index, after deleting a file.
+    """
+    targets = []
+    digest, path, size = render_simple_detail(project, request, store=True)
+    simple_index_part = _payload_targets_part(path, size, digest)
+    targets.append(simple_index_part)
+    if file:
+        file_part = _payload_targets_part(file.path, file.size, file.blake2_256_digest)
+        targets.append(file_part)
+
+    response = requests.post(
+        request.registry.settings["tuf.api.targets.url"], json=_payload(targets)
+    )
+
+    return _handle(response)
+
+
+def delete_file(request, project, file):
+    """Call RSTUF to remove file and project simple index from TUF targets metadata.
+
+    NOTE: Simple index needs to be added separately.
+    """
+    index_path = current_simple_details_path(request, project)
+    targets = [file.path, index_path]
+
+    response = requests.delete(
+        request.registry.settings["tuf.api.targets.url"], json=_payload(targets)
+    )
+
+    return _handle(response)
+
+
+def delete_release(request, release):
+    files = request.db.query(File).filter(File.release_id == release.id).all()
+
+    tasks = []
+    for file in files:
+        tasks.append(delete_file(request, release.project, file))
+
+    return tasks