Add the config option storage.sandbox

This config option can be used to define the directory into which all
sandbox folders will be created. This can be useful if the default
temporary directory of the operating system that would otherwise be
used is not suited, for example because it is on a slower file system or
it has insufficient empty space.

The option is passed to the `SandboxFolder` constructor whenever it is
used by AiiDA's internal code. Since the option requires the presence of
a configuration, the retrieval of the option is added as high up in the
API as possible. This is in order to not couple components to the
configuration that should remain independent.

aiida/cmdline/commands/cmd_archive.py

@@ -367,7 +367,7 @@ def import_archive(
     }
 
     for archive, web_based in all_archives:
-        _import_archive_and_migrate(archive, web_based, import_kwargs, migration)
+        _import_archive_and_migrate(ctx, archive, web_based, import_kwargs, migration)
 
 
 def _echo_exception(msg: str, exception, warn_only: bool = False):
@@ -421,7 +421,9 @@ def _gather_imports(archives, webpages) -> List[Tuple[str, bool]]:
     return final_archives
 
 
-def _import_archive_and_migrate(archive: str, web_based: bool, import_kwargs: dict, try_migration: bool):
+def _import_archive_and_migrate(
+    ctx: click.Context, archive: str, web_based: bool, import_kwargs: dict, try_migration: bool
+):
     """Perform the archive import.
 
     :param archive: the path or URL to the archive
@@ -435,8 +437,9 @@ def _import_archive_and_migrate(archive: str, web_based: bool, import_kwargs: di
     from aiida.tools.archive.imports import import_archive as _import_archive
 
     archive_format = get_format()
+    filepath = ctx.obj['config'].get_option('storage.sandbox') or None
 
-    with SandboxFolder() as temp_folder:
+    with SandboxFolder(filepath=filepath) as temp_folder:
 
         archive_path = archive
 
@@ -462,15 +465,15 @@ def _import_archive_and_migrate(archive: str, web_based: bool, import_kwargs: di
                     new_path = temp_folder.get_abs_path('migrated_archive.aiida')
                     archive_format.migrate(archive_path, new_path, archive_format.latest_version, compression=0)
                     archive_path = new_path
-                except Exception as exception:
-                    _echo_exception(f'an exception occurred while migrating the archive {archive}', exception)
+                except Exception as sub_exception:
+                    _echo_exception(f'an exception occurred while migrating the archive {archive}', sub_exception)
 
                 echo.echo_report('proceeding with import of migrated archive')
                 try:
                     _import_archive(archive_path, archive_format=archive_format, **import_kwargs)
-                except Exception as exception:
+                except Exception as sub_exception:
                     _echo_exception(
-                        f'an exception occurred while trying to import the migrated archive {archive}', exception
+                        f'an exception occurred while trying to import the migrated archive {archive}', sub_exception
                     )
             else:
                 _echo_exception(f'an exception occurred while trying to import the archive {archive}', exception)

aiida/engine/daemon/execmanager.py

@@ -27,6 +27,7 @@
 from aiida.common.datastructures import CalcInfo
 from aiida.common.folders import SandboxFolder
 from aiida.common.links import LinkType
+from aiida.manage.configuration import get_config_option
 from aiida.orm import CalcJobNode, Code, FolderData, Node, RemoteData, load_node
 from aiida.orm.utils.log import get_dblogger_extra
 from aiida.repository.common import FileType
@@ -438,6 +439,7 @@ def retrieve_calculation(calculation: CalcJobNode, transport: Transport, retriev
     """
     logger_extra = get_dblogger_extra(calculation)
     workdir = calculation.get_remote_workdir()
+    filepath_sandbox = get_config_option('storage.sandbox') or None
 
     EXEC_LOGGER.debug(f'Retrieving calc {calculation.pk}', extra=logger_extra)
     EXEC_LOGGER.debug(f'[retrieval of calc {calculation.pk}] chdir {workdir}', extra=logger_extra)
@@ -462,7 +464,7 @@ def retrieve_calculation(calculation: CalcJobNode, transport: Transport, retriev
         retrieve_list = calculation.get_retrieve_list()
         retrieve_temporary_list = calculation.get_retrieve_temporary_list()
 
-        with SandboxFolder() as folder:
+        with SandboxFolder(filepath_sandbox) as folder:
             retrieve_files_from_list(calculation, transport, folder.abspath, retrieve_list)
             # Here I retrieved everything; now I store them inside the calculation
             retrieved_files.base.repository.put_object_from_tree(folder.abspath)

aiida/engine/processes/calcjobs/calcjob.py

@@ -429,11 +429,14 @@ def _perform_import(self):
         from aiida.common.datastructures import CalcJobState
         from aiida.common.folders import SandboxFolder
         from aiida.engine.daemon.execmanager import retrieve_calculation
+        from aiida.manage import get_config_option
         from aiida.transports.plugins.local import LocalTransport
 
+        filepath_sandbox = get_config_option('storage.sandbox') or None
+
         with LocalTransport() as transport:
-            with SandboxFolder() as folder:
-                with SandboxFolder() as retrieved_temporary_folder:
+            with SandboxFolder(filepath_sandbox) as folder:
+                with SandboxFolder(filepath_sandbox) as retrieved_temporary_folder:
                     self.presubmit(folder)
                     self.node.set_remote_workdir(
                         self.inputs.remote_folder.get_remote_path()  # type: ignore[union-attr]

aiida/engine/processes/calcjobs/tasks.py

@@ -72,14 +72,15 @@ async def task_upload_job(process: 'CalcJob', transport_queue: TransportQueue, c
 
     initial_interval = get_config_option(RETRY_INTERVAL_OPTION)
     max_attempts = get_config_option(MAX_ATTEMPTS_OPTION)
+    filepath_sandbox = get_config_option('storage.sandbox') or None
 
     authinfo = node.get_authinfo()
 
     async def do_upload():
         with transport_queue.request_transport(authinfo) as request:
             transport = await cancellable.with_interrupt(request)
 
-            with SandboxFolder() as folder:
+            with SandboxFolder(filepath_sandbox) as folder:
                 # Any exception thrown in `presubmit` call is not transient so we circumvent the exponential backoff
                 try:
                     calc_info = process.presubmit(folder)

aiida/manage/configuration/schema/config-v8.schema.json

@@ -130,6 +130,10 @@
           "minimum": 1,
           "description": "Timeout in seconds for communications with RabbitMQ"
         },
+        "storage.sandbox": {
+          "type": "string",
+          "description": "Absolute path to the directory to store sandbox folders."
+        },
         "caching.default_enabled": {
           "type": "boolean",
           "default": false,

aiida/orm/nodes/repository.py

@@ -8,6 +8,7 @@
 from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, Iterator, List, Optional, TextIO, Tuple, Union
 
 from aiida.common import exceptions
+from aiida.manage import get_config_option
 from aiida.repository import File, Repository
 from aiida.repository.backend import SandboxRepositoryBackend
 
@@ -80,7 +81,8 @@ def _repository(self) -> Repository:
                 backend = self._node.backend.get_repository()
                 self._repository_instance = Repository.from_serialized(backend=backend, serialized=self.metadata)
             else:
-                self._repository_instance = Repository(backend=SandboxRepositoryBackend())
+                filepath = get_config_option('storage.sandbox') or None
+                self._repository_instance = Repository(backend=SandboxRepositoryBackend(filepath))
 
         return self._repository_instance
 

aiida/repository/backend/sandbox.py

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 """Implementation of the ``AbstractRepositoryBackend`` using a sandbox folder on disk as the backend."""
+from __future__ import annotations
+
 import contextlib
 import os
 import shutil
-from typing import BinaryIO, Iterable, Iterator, List, Optional, Tuple
+import typing as t
 import uuid
 
 from aiida.common.folders import SandboxFolder
@@ -16,8 +18,13 @@
 class SandboxRepositoryBackend(AbstractRepositoryBackend):
     """Implementation of the ``AbstractRepositoryBackend`` using a sandbox folder on disk as the backend."""
 
-    def __init__(self):
-        self._sandbox: Optional[SandboxFolder] = None
+    def __init__(self, filepath: str | None = None):
+        """Construct a new instance.
+
+        :param filepath: The path to the directory in which the sandbox folder should be created.
+        """
+        self._sandbox: SandboxFolder | None = None
+        self._filepath: str | None = filepath
 
     def __str__(self) -> str:
         """Return the string representation of this repository."""
@@ -30,15 +37,15 @@ def __del__(self):
         self.erase()
 
     @property
-    def uuid(self) -> Optional[str]:
+    def uuid(self) -> str | None:
         """Return the unique identifier of the repository.
 
         .. note:: A sandbox folder does not have the concept of a unique identifier and so always returns ``None``.
         """
         return None
 
     @property
-    def key_format(self) -> Optional[str]:
+    def key_format(self) -> str | None:
         return 'uuid4'
 
     def initialise(self, **kwargs) -> None:
@@ -58,7 +65,7 @@ def is_initialised(self) -> bool:
     def sandbox(self):
         """Return the sandbox instance of this repository."""
         if self._sandbox is None:
-            self._sandbox = SandboxFolder()
+            self._sandbox = SandboxFolder(filepath=self._filepath)
 
         return self._sandbox
 
@@ -72,7 +79,7 @@ def erase(self):
             finally:
                 self._sandbox = None
 
-    def _put_object_from_filelike(self, handle: BinaryIO) -> str:
+    def _put_object_from_filelike(self, handle: t.BinaryIO) -> str:
         """Store the byte contents of a file in the repository.
 
         :param handle: filelike object with the byte content to be stored.
@@ -87,31 +94,31 @@ def _put_object_from_filelike(self, handle: BinaryIO) -> str:
 
         return key
 
-    def has_objects(self, keys: List[str]) -> List[bool]:
+    def has_objects(self, keys: list[str]) -> list[bool]:
         result = []
         dirlist = os.listdir(self.sandbox.abspath)
         for key in keys:
             result.append(key in dirlist)
         return result
 
     @contextlib.contextmanager
-    def open(self, key: str) -> Iterator[BinaryIO]:
+    def open(self, key: str) -> t.Iterator[t.BinaryIO]:
         super().open(key)
 
         with self.sandbox.open(key, mode='rb') as handle:
             yield handle
 
-    def iter_object_streams(self, keys: List[str]) -> Iterator[Tuple[str, BinaryIO]]:
+    def iter_object_streams(self, keys: list[str]) -> t.Iterator[tuple[str, t.BinaryIO]]:
         for key in keys:
             with self.open(key) as handle:  # pylint: disable=not-context-manager
                 yield key, handle
 
-    def delete_objects(self, keys: List[str]) -> None:
+    def delete_objects(self, keys: list[str]) -> None:
         super().delete_objects(keys)
         for key in keys:
             os.remove(os.path.join(self.sandbox.abspath, key))
 
-    def list_objects(self) -> Iterable[str]:
+    def list_objects(self) -> t.Iterable[str]:
         return self.sandbox.get_content_list()
 
     def maintain(self, dry_run: bool = False, live: bool = True, **kwargs) -> None:

aiida/storage/sqlite_temp/backend.py

@@ -19,7 +19,7 @@
 from sqlalchemy.orm import Session
 
 from aiida.common.exceptions import ClosedStorage
-from aiida.manage import Profile
+from aiida.manage import Profile, get_config_option
 from aiida.manage.configuration.settings import AIIDA_CONFIG_FOLDER
 from aiida.orm.entities import EntityTypes
 from aiida.orm.implementation import BackendEntity, StorageBackend
@@ -133,7 +133,7 @@ def get_repository(self) -> SandboxRepositoryBackend:
             raise ClosedStorage(str(self))
         if self._repo is None:
             # to-do this does not seem to be removing the folder on garbage collection?
-            self._repo = SandboxRepositoryBackend()
+            self._repo = SandboxRepositoryBackend(filepath=get_config_option('storage.sandbox') or None)
         return self._repo
 
     @property