diff --git a/.coveragerc b/.coveragerc index f0a88153d..d3c79b140 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,6 @@ omit = **/migrations/* - **/management/commands/* **/settings/* **/wsgi.py **/tests/* diff --git a/storage_service/common/management/commands/populate_aip_stored_dates.py b/storage_service/common/management/commands/populate_aip_stored_dates.py index 4e68517f1..8927b58f4 100644 --- a/storage_service/common/management/commands/populate_aip_stored_dates.py +++ b/storage_service/common/management/commands/populate_aip_stored_dates.py @@ -12,18 +12,15 @@ import os from datetime import datetime -import pytz from common.management.commands import StorageServiceCommand -from django.conf import settings from django.core.management.base import CommandError +from django.utils.timezone import get_current_timezone from locations.models.package import Package from locations.models.package import Space # Suppress the logging from models/package.py. logging.config.dictConfig({"version": 1, "disable_existing_loggers": True}) -TIMEZONE = pytz.timezone(settings.TIME_ZONE) - class Command(StorageServiceCommand): help = __doc__ @@ -55,6 +52,8 @@ def handle(self, *args, **options): success_count = 0 skipped_count = 0 + tz = get_current_timezone() + for aip in aips: # Skip AIPs that already have datestamps. if aip.stored_date is not None: @@ -71,7 +70,7 @@ def handle(self, *args, **options): ) continue - aip.stored_date = datetime.fromtimestamp(int(modified_unix), tz=TIMEZONE) + aip.stored_date = datetime.fromtimestamp(int(modified_unix), tz=tz) aip.save() success_count += 1 diff --git a/tests/common/conftest.py b/tests/common/conftest.py new file mode 100644 index 000000000..7cf1f6129 --- /dev/null +++ b/tests/common/conftest.py @@ -0,0 +1,12 @@ +import pytest + + +@pytest.fixture(autouse=True) +def do_not_reset_the_root_logger(mocker): + # Some management commands suppress logging from models/package.py + # by calling logging.config.dictConfig directly which breaks the caplog + # fixture. See https://github.com/pytest-dev/pytest/discussions/11011 + # + # This avoids breaking the caplog fixture when those management command + # modules are imported or called during tests. + mocker.patch("logging.config") diff --git a/tests/common/test_command_import_aip.py b/tests/common/test_command_import_aip.py index c90ea8e4f..1d7acef71 100644 --- a/tests/common/test_command_import_aip.py +++ b/tests/common/test_command_import_aip.py @@ -41,7 +41,6 @@ def test_import_aip_command_creates_uncompressed_package( ): mocker.patch("os.chown") mocker.patch("pwd.getpwnam") - mocker.patch("logging.config") call_command( "import_aip", "--decompress-source", diff --git a/tests/common/test_command_populate_aip_stored_dates.py b/tests/common/test_command_populate_aip_stored_dates.py new file mode 100644 index 000000000..d24215cfb --- /dev/null +++ b/tests/common/test_command_populate_aip_stored_dates.py @@ -0,0 +1,212 @@ +import datetime +import pathlib +from typing import Any +from unittest import mock + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError +from locations import models + + +@pytest.fixture +@pytest.mark.django_db +def fs_space(tmp_path: pathlib.Path) -> models.Location: + space_dir = tmp_path / "space" + space_dir.mkdir() + + staging_dir = tmp_path / "staging" + staging_dir.mkdir() + + result = models.Space.objects.create( + access_protocol=models.Space.LOCAL_FILESYSTEM, + path=str(space_dir), + staging_path=str(staging_dir), + ) + models.LocalFilesystem.objects.create(space=result) + + return result + + +@pytest.fixture +@pytest.mark.django_db +def aip_storage_fs_location(fs_space: models.Space) -> models.Location: + return models.Location.objects.create( + space=fs_space, + purpose=models.Location.AIP_STORAGE, + relative_path="fs-aips", + ) + + +@pytest.fixture +@pytest.mark.django_db +def aip_deleted_fs_location(aip_storage_fs_location: models.Location) -> models.Package: + return models.Package.objects.create( + package_type=models.Package.AIP, + status=models.Package.DELETED, + current_location=aip_storage_fs_location, + current_path="deleted.7z", + ) + + +@pytest.fixture +@pytest.mark.django_db +def aip_uploaded_fs_location( + aip_storage_fs_location: models.Location, +) -> models.Package: + return models.Package.objects.create( + package_type=models.Package.AIP, + status=models.Package.UPLOADED, + current_location=aip_storage_fs_location, + current_path="uploaded.7z", + ) + + +@pytest.fixture +@pytest.mark.django_db +def secondary_fs_space(tmp_path: pathlib.Path) -> models.Location: + space_dir = tmp_path / "secondary-space" + space_dir.mkdir() + + staging_dir = tmp_path / "secondary-staging" + staging_dir.mkdir() + + result = models.Space.objects.create( + access_protocol=models.Space.LOCAL_FILESYSTEM, + path=str(space_dir), + staging_path=str(staging_dir), + ) + models.LocalFilesystem.objects.create(space=result) + + return result + + +@pytest.fixture +@pytest.mark.django_db +def secondary_aip_storage_fs_location( + secondary_fs_space: models.Space, +) -> models.Location: + return models.Location.objects.create( + space=secondary_fs_space, + purpose=models.Location.AIP_STORAGE, + relative_path="secondary-fs-aips", + ) + + +@pytest.fixture +@pytest.mark.django_db +def secondary_aip_uploaded_fs_location( + secondary_aip_storage_fs_location: models.Location, +) -> models.Package: + return models.Package.objects.create( + package_type=models.Package.AIP, + status=models.Package.UPLOADED, + current_location=secondary_aip_storage_fs_location, + current_path="secondary-uploaded.7z", + ) + + +@pytest.mark.django_db +def test_command_fails_when_there_are_no_uploaded_aips( + aip_deleted_fs_location: models.Package, +) -> None: + with pytest.raises(CommandError, match="No AIPs with status UPLOADED found"): + call_command("populate_aip_stored_dates") + + +@pytest.mark.django_db +@mock.patch("common.management.commands.StorageServiceCommand.error") +@mock.patch("common.management.commands.StorageServiceCommand.success") +def test_command_completes_when_location_does_not_contain_aips( + success: mock.Mock, + error: mock.Mock, + aip_uploaded_fs_location: models.Package, + secondary_aip_storage_fs_location: models.Location, +) -> None: + call_command( + "populate_aip_stored_dates", + "--location-uuid", + secondary_aip_storage_fs_location.uuid, + ) + + success.assert_called_once_with("Complete. No matching AIPs found.") + error.assert_not_called() + + +@pytest.mark.django_db +@mock.patch("os.path.getmtime", side_effect=[1710831600]) +@mock.patch("common.management.commands.StorageServiceCommand.error") +@mock.patch("common.management.commands.StorageServiceCommand.success") +def test_command_filters_aips_by_location_uuid( + success: mock.Mock, + error: mock.Mock, + getmtime: mock.Mock, + settings: Any, + aip_uploaded_fs_location: models.Package, + secondary_aip_uploaded_fs_location: models.Package, + secondary_aip_storage_fs_location: models.Location, +) -> None: + settings.TIME_ZONE = "UTC" + + call_command( + "populate_aip_stored_dates", + "--location-uuid", + secondary_aip_storage_fs_location.uuid, + ) + + assert models.Package.objects.get( + uuid=secondary_aip_uploaded_fs_location.uuid + ).stored_date == datetime.datetime(2024, 3, 19, 7, 0, tzinfo=datetime.timezone.utc) + + getmtime.assert_called_once_with(secondary_aip_uploaded_fs_location.full_path) + success.assert_called_once_with( + "Complete. Datestamps for 1 of 1 identified AIPs added. 0 AIPs that already have stored_dates were skipped." + ) + error.assert_not_called() + + +@pytest.mark.django_db +@mock.patch("os.path.getmtime", side_effect=OSError("no such file or directory")) +@mock.patch("common.management.commands.StorageServiceCommand.error") +@mock.patch("common.management.commands.StorageServiceCommand.success") +def test_command_logs_error_when_it_cannot_read_aip_file( + success: mock.Mock, + error: mock.Mock, + getmtime: mock.Mock, + secondary_aip_uploaded_fs_location: models.Package, + secondary_aip_storage_fs_location: models.Location, +) -> None: + call_command( + "populate_aip_stored_dates", + "--location-uuid", + secondary_aip_storage_fs_location.uuid, + ) + + getmtime.assert_called_once_with(secondary_aip_uploaded_fs_location.full_path) + success.assert_called_once_with( + "Complete. Datestamps for 0 of 1 identified AIPs added. 0 AIPs that already have stored_dates were skipped." + ) + error.assert_called_once_with( + f"Unable to get timestamp for local AIP {secondary_aip_uploaded_fs_location.uuid}. Details: no such file or directory" + ) + + +@pytest.mark.django_db +@mock.patch("common.management.commands.StorageServiceCommand.error") +@mock.patch("common.management.commands.StorageServiceCommand.success") +def test_command_skips_aips_with_stored_dates( + success: mock.Mock, + error: mock.Mock, + secondary_aip_uploaded_fs_location: models.Package, +) -> None: + secondary_aip_uploaded_fs_location.stored_date = datetime.datetime( + 2023, 1, 1, 0, 0, tzinfo=datetime.timezone.utc + ) + secondary_aip_uploaded_fs_location.save() + + call_command("populate_aip_stored_dates") + + success.assert_called_once_with( + "Complete. All 1 AIPs that already have stored_dates skipped." + ) + error.assert_not_called()