Add possibility to override configs

.gitignore

@@ -1,3 +1,7 @@
+# Ahcore
+additional_config/*
+scripts/*
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

ahcore/callbacks.py

@@ -378,7 +378,7 @@ def _batch_end(
             tile_overlap = inference_grid.tile_overlap
 
             # TODO: We are really putting strange things in the Queue if we may believe mypy
-            new_queue: Queue[Any] = Queue()
+            new_queue: Queue[Any] = Queue()   # pylint: disable=unsubscriptable-object
             parent_conn, child_conn = Pipe()
             new_writer = H5FileImageWriter(
                 output_filename,

ahcore/cli/data.py

@@ -58,7 +58,7 @@ def copy_data(args: argparse.Namespace) -> None:
 
 
 def register_parser(
-    parser: argparse._SubParsersAction[Any],
+    parser: argparse._SubParsersAction[Any],  # pylint: disable=unsubscriptable-object
 ) -> None:  # pylint: disable=E1136
     """Register inspect commands to a root parser."""
     data_parser = parser.add_parser("data", help="Data utilities")

ahcore/hydra_plugins/__init__.py

@@ -0,0 +1,4 @@
+"""Hydra plugins for ahcore."""
+from .additional_configs import register_additional_config_search_path
+
+__all__ = ("register_additional_config_search_path",)

ahcore/hydra_plugins/additional_configs.py

@@ -0,0 +1,53 @@
+import warnings
+from pathlib import Path
+
+from hydra.core.config_search_path import ConfigSearchPath
+from hydra.core.plugins import Plugins
+from hydra.plugins.search_path_plugin import SearchPathPlugin
+
+from ahcore.exceptions import ConfigurationError
+from ahcore.utils.io import get_logger
+
+logger = get_logger(__name__)
+
+
+class AdditionalSearchPathPlugin(SearchPathPlugin):
+    """This plugin allows to overwrite the ahcore configurations without needed to fork the repository."""
+
+    def manipulate_search_path(self, search_path: ConfigSearchPath) -> None:
+        additional_path = Path(__file__).parent.parent.parent / "additional_config"
+        if additional_path.is_file():
+            raise ConfigurationError("Found additional_config file, but expected a folder.")
+
+        elif additional_path.is_dir():
+            if not list(additional_path.glob("*")):
+                warnings.warn(
+                    f"Found additional_config folder in {additional_path}, without any configuration files. "
+                    "If you want to overwrite the default ahcore configs, "
+                    "please add these to the additional_config folder. "
+                    "You can symlink your additional configuration to this folder. "
+                    "See the documentation at https://docs.aiforoncology.nl/ahcore/configuration.html "
+                    "for more information."
+                )
+            else:
+                # Add additional search path for configs
+                logger.info(f"Adding additional search path for configs: file://{additional_path}")
+                search_path.prepend(provider="hydra-ahcore", path=f"file://{additional_path}")
+        else:
+            logger.info(
+                "No additional_config folder found. Will use standard ahcore configurations."
+                "If you want to overwrite or extend the default ahcore configs, you can add these to the "
+                "additional_config folder. You could also symlink your additional configuration to this folder."
+                "See the documentation at https://docs.aiforoncology.nl/ahcore/configuration.html."
+            )
+
+
+def register_additional_config_search_path() -> None:
+    """
+    Register the additional_config folder as a search path for hydra.
+
+    Returns
+    -------
+    None
+    """
+    Plugins.instance().register(AdditionalSearchPathPlugin)

docs/conf.py

@@ -77,8 +77,8 @@
 
 # General information about the project.
 project = "ahcore"
-copyright = "2022, ahcore contributors"
-author = "Jonas Teuwen"
+copyright = "2023, ahcore contributors"
+author = "AI for Oncology"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/configuration.rst

@@ -0,0 +1,9 @@
+Configuration
+=============
+
+Ahcore is configured using Hydra. In the ahcore main folder there is a folder `config`,
+these contain the standard configuration files. The configuration files are in YAML format according
+to the Hydra specification.
+
+If you want to add your own configuration files, you can do so by creating a new folder in the `additional_config`
+folder. The configuration files here will override the standard configuration files.

docs/index.rst

@@ -1,7 +1,7 @@
 .. role:: bash(code)
    :language: bash
 
-AI for Oncology Core for Comptuational Pathology
+AI for Oncology Core for Computational Pathology
 ================================================
 
 
@@ -10,6 +10,7 @@ AI for Oncology Core for Comptuational Pathology
    :caption: Contents:
 
    cli
+   configuration
    contributing
    modules
 

pyproject.toml

@@ -43,3 +43,49 @@ max-line-length = "120"
 max-args=20
 max-branches=30
 max-parents=15
+
+[tool.ruff]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+
+line-length = 120
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.10
+target-version = "py310"
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10

setup.py

@@ -61,7 +61,7 @@
             "pytest",
             "numpydoc",
             "pylint==2.17.7",
-            "black==23.9.1",
+            "black==23.10.1",
             "types-Pillow",
             "sphinx",
             "sphinx_copybutton",

tools/inference.py

@@ -2,10 +2,12 @@
 import hydra
 from omegaconf import DictConfig
 
-# load environment variables from `.env` file if it exists
-# recursively searches for `.env` in all folders starting from work dir
 dotenv.load_dotenv(override=True)
 
+from ahcore.hydra_plugins import register_additional_config_search_path  # noqa: E402
+
+register_additional_config_search_path()
+
 
 @hydra.main(
     config_path="../config",

tools/train.py

@@ -2,10 +2,12 @@
 import hydra
 from omegaconf import DictConfig
 
-# load environment variables from `.env` file if it exists
-# recursively searches for `.env` in all folders starting from work dir
 dotenv.load_dotenv(override=True)
 
+from ahcore.hydra_plugins import register_additional_config_search_path  # noqa: E402
+
+register_additional_config_search_path()
+
 
 @hydra.main(
     config_path="../config",