Merge branch 'artefactual:qa/0.x' into dev/issue-1622-common-utils-te…

…sts-replace-os-path-with-pathlib

requirements-dev.txt

@@ -12,9 +12,9 @@ asgiref==3.7.2
     #   django
 bagit==1.8.1
     # via -r requirements.txt
-boto3==1.34.55
+boto3==1.34.60
     # via -r requirements.txt
-botocore==1.34.55
+botocore==1.34.60
     # via
     #   -r requirements.txt
     #   boto3
@@ -106,11 +106,11 @@ idna==3.6
     # via
     #   -r requirements.txt
     #   requests
-importlib-metadata==7.0.1
+importlib-metadata==7.0.2
     # via
     #   build
     #   pytest-randomly
-importlib-resources==6.1.2
+importlib-resources==6.1.3
     # via -r requirements.txt
 iniconfig==2.0.0
     # via pytest
@@ -142,7 +142,7 @@ lxml==5.1.0
     #   sword2
 metsrw==0.5.1
     # via -r requirements.txt
-mozilla-django-oidc==4.0.0
+mozilla-django-oidc==4.0.1
     # via -r requirements.txt
 msgpack==1.0.8
     # via
@@ -184,7 +184,7 @@ oslo-utils==7.1.0
     #   -r requirements.txt
     #   oslo-serialization
     #   python-keystoneclient
-packaging==23.2
+packaging==24.0
     # via
     #   -r requirements.txt
     #   build
@@ -203,7 +203,7 @@ pbr==6.0.0
     #   oslo-serialization
     #   python-keystoneclient
     #   stevedore
-pip-tools==7.4.0
+pip-tools==7.4.1
     # via -r requirements-dev.in
 platformdirs==4.2.0
     # via
@@ -230,11 +230,11 @@ pycparser==2.21
     # via
     #   -r requirements.txt
     #   cffi
-pyopenssl==24.0.0
+pyopenssl==24.1.0
     # via
     #   -r requirements.txt
     #   josepy
-pyparsing==3.1.1
+pyparsing==3.1.2
     # via
     #   -r requirements.txt
     #   httplib2
@@ -245,7 +245,7 @@ pyproject-hooks==1.0.0
     # via
     #   build
     #   pip-tools
-pytest==8.0.2
+pytest==8.1.1
     # via
     #   -r requirements-dev.in
     #   pytest-cov
@@ -332,7 +332,7 @@ tomli==2.0.1
     #   pyproject-hooks
     #   pytest
     #   tox
-tox==4.13.0
+tox==4.14.1
     # via -r requirements-dev.in
 typing-extensions==4.10.0
     # via
@@ -351,7 +351,7 @@ urllib3==1.26.18
     #   requests
 virtualenv==20.25.1
     # via tox
-wheel==0.42.0
+wheel==0.43.0
     # via pip-tools
 whitenoise==6.6.0
     # via -r requirements.txt

requirements.txt

@@ -10,9 +10,9 @@ asgiref==3.7.2
     # via django
 bagit==1.8.1
     # via -r requirements.in
-boto3==1.34.55
+boto3==1.34.60
     # via -r requirements.in
-botocore==1.34.55
+botocore==1.34.60
     # via
     #   boto3
     #   s3transfer
@@ -67,7 +67,7 @@ httplib2==0.22.0
     # via sword2
 idna==3.6
     # via requests
-importlib-resources==6.1.2
+importlib-resources==6.1.3
     # via -r requirements.in
 iso8601==2.1.0
     # via
@@ -91,7 +91,7 @@ lxml==5.1.0
     #   sword2
 metsrw==0.5.1
     # via -r requirements.in
-mozilla-django-oidc==4.0.0
+mozilla-django-oidc==4.0.1
     # via -r requirements.in
 msgpack==1.0.8
     # via oslo-serialization
@@ -118,7 +118,7 @@ oslo-utils==7.1.0
     # via
     #   oslo-serialization
     #   python-keystoneclient
-packaging==23.2
+packaging==24.0
     # via
     #   gunicorn
     #   oslo-utils
@@ -143,9 +143,9 @@ pyasn1-modules==0.3.0
     # via python-ldap
 pycparser==2.21
     # via cffi
-pyopenssl==24.0.0
+pyopenssl==24.1.0
     # via josepy
-pyparsing==3.1.1
+pyparsing==3.1.2
     # via
     #   httplib2
     #   oslo-utils

storage_service/locations/api/resources.py

@@ -1380,7 +1380,7 @@ def extract_file_request(self, request, bundle, **kwargs):
             )
 
         response = utils.download_file_stream(extracted_file_path, temp_dir)
-
+        package.clear_local_tempdirs()
         return response
 
     @_custom_endpoint(expected_methods=["get", "head"])

storage_service/locations/migrations/0037_django42.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.5 on 2023-09-14 19:23
+# Generated by Django 4.2.10 on 2024-03-07 19:10
 from django.db import migrations
 from django.db import models
 
@@ -12,8 +12,6 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name="package",
             name="related_packages",
-            field=models.ManyToManyField(
-                related_name="related", to="locations.package"
-            ),
+            field=models.ManyToManyField(to="locations.package"),
         ),
     ]

storage_service/locations/models/package.py

@@ -133,7 +133,7 @@ class Package(models.Model):
         (DEPOSIT, _("FEDORA Deposit")),
     )
     package_type = models.CharField(max_length=8, choices=PACKAGE_TYPE_CHOICES)
-    related_packages = models.ManyToManyField("self", related_name="related")
+    related_packages = models.ManyToManyField("self")
 
     PENDING = "PENDING"
     STAGING = "STAGING"

storage_service/locations/models/s3.py

@@ -3,6 +3,7 @@
 import pprint
 import re
 from functools import wraps
+from urllib.parse import urlparse
 
 import boto3
 import botocore
@@ -75,10 +76,11 @@ def resource(self):
             )
             boto_args = {
                 "service_name": "s3",
-                "endpoint_url": self.endpoint_url,
                 "region_name": self.region,
                 "config": config,
             }
+            if not self._is_global_endpoint(self.endpoint_url):
+                boto_args["endpoint_url"] = self.endpoint_url
             if self.access_key_id and self.secret_access_key:
                 boto_args.update(
                     aws_access_key_id=self.access_key_id,
@@ -87,6 +89,9 @@ def resource(self):
             self._resource = boto3.resource(**boto_args)
         return self._resource
 
+    def _is_global_endpoint(self, url):
+        return urlparse(url).netloc == "s3.amazonaws.com"
+
     @boto_exception
     def _ensure_bucket_exists(self):
         """Ensure that the bucket exists by asking it something about itself.

tests/locations/test_api.py

@@ -7,6 +7,7 @@
 from unittest import mock
 from urllib.parse import urlparse
 
+import pytest
 from administration import roles
 from django.contrib.auth.models import User
 from django.test import TestCase
@@ -1095,3 +1096,111 @@ def test_pipeline_create(self):
         assert pipeline.parse_and_fix_url(pipeline.remote_name) == urlparse(
             "http://192.168.0.10"
         )
+
+
+@pytest.fixture
+def internal_processing_location(db, tmp_path):
+    space_dir = tmp_path / "space"
+    space_dir.mkdir()
+
+    staging_dir = space_dir / "staging"
+    staging_dir.mkdir()
+
+    return models.Location.objects.create(
+        space=models.Space.objects.create(
+            access_protocol=models.Space.LOCAL_FILESYSTEM,
+            path=space_dir,
+            staging_path=staging_dir,
+        ),
+        purpose=models.Location.STORAGE_SERVICE_INTERNAL,
+        relative_path=staging_dir.relative_to(space_dir),
+    )
+
+
+@pytest.fixture
+def aip_storage_location(db):
+    space = models.Space.objects.create(
+        access_protocol=models.Space.S3,
+    )
+    models.S3.objects.create(space=space)
+
+    return models.Location.objects.create(
+        space=space,
+        purpose=models.Location.AIP_STORAGE,
+        relative_path="aips",
+    )
+
+
+@pytest.fixture
+def compressed_bag_fixture_path():
+    return FIXTURES_DIR / "working_bag.zip"
+
+
+@pytest.fixture
+def s3_resource(compressed_bag_fixture_path, aip_storage_location):
+    """Mock the S3 bucket interactions in S3.move_to_storage_service."""
+
+    def download_file(_key, dest_file):
+        shutil.copy(compressed_bag_fixture_path, dest_file)
+
+    return mock.Mock(
+        **{
+            "Bucket.side_effect": [
+                mock.Mock(
+                    **{
+                        "download_file.side_effect": download_file,
+                    }
+                ),
+                mock.Mock(
+                    **{
+                        "objects.filter.return_value": [
+                            mock.Mock(
+                                key=f"{aip_storage_location.relative_path}/{compressed_bag_fixture_path.name}"
+                            )
+                        ]
+                    }
+                ),
+            ]
+        }
+    )
+
+
+@mock.patch("boto3.resource")
+def test_s3_space_deletes_temporary_files_after_extracting_file(
+    resource,
+    admin_client,
+    compressed_bag_fixture_path,
+    s3_resource,
+    aip_storage_location,
+    internal_processing_location,
+):
+    # Mock the S3 bucket interactions.
+    resource.side_effect = [s3_resource]
+
+    # Add a compressed AIP to the AIP Storage location.
+    package = models.Package.objects.create(
+        current_location=aip_storage_location,
+        package_type=models.Package.AIP,
+        current_path=compressed_bag_fixture_path.name,
+    )
+
+    # Extract a file from the compressed AIP.
+    response = admin_client.get(
+        reverse(
+            "extract_file_request",
+            kwargs={"api_name": "v2", "resource_name": "file", "uuid": package.uuid},
+        ),
+        {"relative_path_to_file": "working_bag/data/test.txt"},
+    )
+
+    # Verify the response attributes.
+    assert response.status_code == 200
+    assert response["content-type"] == "text/plain"
+    assert response["content-disposition"] == 'attachment; filename="test.txt"'
+
+    # Verify the contents of the extracted file.
+    result = b"".join(response.streaming_content)
+    assert result.decode() == "test"
+
+    # Verify there are no temporary files left in the internal processing location.
+    assert list(pathlib.Path(internal_processing_location.full_path).iterdir()) == []