diff --git a/.github/workflows/hermes-zenodo.yml b/.github/workflows/hermes-zenodo.yml index 402c5944..302fdeb0 100644 --- a/.github/workflows/hermes-zenodo.yml +++ b/.github/workflows/hermes-zenodo.yml @@ -42,7 +42,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.10' - - run: pip install hermes + - run: pip install hermes hermes-plugin-python - run: hermes harvest - run: hermes process - run: hermes curate diff --git a/.gitignore b/.gitignore index d7c36ebe..5856425f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ dist/ # HERMES workflow specifics .hermes/ +hermes.log diff --git a/CITATION.cff b/CITATION.cff index 36db497f..dfb7efe6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -14,7 +14,7 @@ title: hermes message: >- If you use this software, please cite it using the metadata from this file. -version: 0.8.1b1 +version: 0.8.1 license: "Apache-2.0" abstract: "Tool to automate software publication. Not stable yet." type: software diff --git a/hermes.toml b/hermes.toml index f455746a..3aa44a8f 100644 --- a/hermes.toml +++ b/hermes.toml @@ -3,10 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 [harvest] -sources = [ "cff" ] # ordered priority (first one is most important) - -[harvest.cff] -enable_validation = false +sources = [ "cff", "toml" ] # ordered priority (first one is most important) [deposit] target = "invenio_rdm" diff --git a/pyproject.toml b/pyproject.toml index 41bad489..42a4747b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ [tool.poetry] # Reference at https://python-poetry.org/docs/pyproject/ name = "hermes" -version = "0.1.0" +version = "0.8.1" description = "Workflow to publish research software with rich metadata" homepage = "https://software-metadata.pub" license = "Apache-2.0" @@ -20,6 +20,7 @@ authors = [ "Jeffrey Kelling ", "Oliver Knodel ", "David Pape ", + "Sophie Kernchen ", ] readme = "README.md" diff --git a/src/hermes/commands/base.py b/src/hermes/commands/base.py index dea22412..82692975 100644 --- a/src/hermes/commands/base.py +++ b/src/hermes/commands/base.py @@ -43,6 +43,7 @@ def __init__(self, parser: argparse.ArgumentParser): self.settings = None self.log = logging.getLogger(f"hermes.{self.command_name}") + self.errors = [] def init_plugins(self): """Collect and initialize the plugins available for the HERMES command.""" diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py index e13cfcf0..542ff891 100644 --- a/src/hermes/commands/cli.py +++ b/src/hermes/commands/cli.py @@ -9,7 +9,9 @@ This module provides the main entry point for the HERMES command line application. """ import argparse +import sys +from hermes import logger from hermes.commands import HermesHelpCommand, HermesCleanCommand, HermesHarvestCommand, HermesProcessCommand, \ HermesCurateCommand, HermesDepositCommand, HermesPostprocessCommand from hermes.commands.base import HermesCommand @@ -56,6 +58,28 @@ def main() -> None: # Actually parse the command line, configure it and execute the selected sub-command. args = parser.parse_args() - args.command.load_settings(args) - args.command.patch_settings(args) - args.command(args) + logger.init_logging() + log = logger.getLogger("hermes.cli") + log.debug("Running hermes with the following command line arguments: %s", args) + + try: + log.debug("Loading settings...") + args.command.load_settings(args) + + log.debug("Update settings from command line...") + args.command.patch_settings(args) + + log.info("Run subcommand %s", args.command.command_name) + args.command(args) + except Exception as e: + log.error("An error occurred during execution of %s", args.command.command_name) + log.debug("Original exception was: %s", e) + + sys.exit(2) + + if args.command.errors: + for e in args.command.errors: + log.error(e) + sys.exit(1) + + sys.exit(0) diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py index e92249ad..0a8e10f5 100644 --- a/src/hermes/commands/deposit/base.py +++ b/src/hermes/commands/deposit/base.py @@ -120,7 +120,6 @@ def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: def __call__(self, args: argparse.Namespace) -> None: self.args = args plugin_name = self.settings.target - print(self.args) ctx = CodeMetaContext() codemeta_file = ctx.get_cache("curate", ctx.hermes_name) @@ -135,11 +134,13 @@ def __call__(self, args: argparse.Namespace) -> None: try: plugin_func = self.plugins[plugin_name](self, ctx) - except KeyError: + except KeyError as e: self.log.error("Plugin '%s' not found.", plugin_name) + self.errors.append(e) try: plugin_func(self) except HermesValidationError as e: self.log.error("Error while executing %s: %s", plugin_name, e) + self.errors.append(e) diff --git a/src/hermes/commands/deposit/invenio.py b/src/hermes/commands/deposit/invenio.py index 6c8ff139..ddee7e68 100644 --- a/src/hermes/commands/deposit/invenio.py +++ b/src/hermes/commands/deposit/invenio.py @@ -105,7 +105,7 @@ class InvenioResolver: invenio_client_class = InvenioClient def __init__(self, client=None): - self.client = client or self.invenio_client_class() + self.client = client or self.invenio_client_class(InvenioDepositSettings()) def resolve_latest_id( self, record_id=None, doi=None, codemeta_identifier=None diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py index dee745c5..4d2a1731 100644 --- a/src/hermes/commands/harvest/base.py +++ b/src/hermes/commands/harvest/base.py @@ -48,10 +48,8 @@ def __call__(self, args: argparse.Namespace) -> None: try: plugin_func = self.plugins[plugin_name]() harvested_data, tags = plugin_func(self) - print(harvested_data) - with HermesHarvestContext( - ctx, plugin_name - ) as harvest_ctx: + + with HermesHarvestContext(ctx, plugin_name) as harvest_ctx: harvest_ctx.update_from(harvested_data, plugin=plugin_name, timestamp=datetime.now().isoformat(), **tags) @@ -59,8 +57,10 @@ def __call__(self, args: argparse.Namespace) -> None: if any(v != _value and t == _tag for v, t in _trace): raise MergeError(_key, None, _value) - except KeyError: + except KeyError as e: self.log.error("Plugin '%s' not found.", plugin_name) + self.errors.append(e) except HermesValidationError as e: self.log.error("Error while executing %s: %s", plugin_name, e) + self.errors.append(e) diff --git a/src/hermes/commands/postprocess/invenio.py b/src/hermes/commands/postprocess/invenio.py index 7d039ae1..a7ba6b53 100644 --- a/src/hermes/commands/postprocess/invenio.py +++ b/src/hermes/commands/postprocess/invenio.py @@ -24,7 +24,7 @@ def config_record_id(ctx): conf.deposit.invenio.record_id = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) except KeyError: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from None + raise RuntimeError("No deposit.invenio configuration available to store record id in") def cff_doi(ctx): diff --git a/src/hermes/commands/postprocess/invenio_rdm.py b/src/hermes/commands/postprocess/invenio_rdm.py index 7b2bf37c..9553f47b 100644 --- a/src/hermes/commands/postprocess/invenio_rdm.py +++ b/src/hermes/commands/postprocess/invenio_rdm.py @@ -23,4 +23,4 @@ def config_record_id(ctx): conf['deposit']['invenio_rdm']['record_id'] = deposition['record_id'] toml.dump(conf, open('hermes.toml', 'w')) except KeyError: - raise RuntimeError("No deposit.invenio configuration available to store record id in") from None + raise RuntimeError("No deposit.invenio_rdm configuration available to store record id in") diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py index 8abeacc5..9e29d1e6 100644 --- a/src/hermes/commands/process/base.py +++ b/src/hermes/commands/process/base.py @@ -58,7 +58,8 @@ def __call__(self, args: argparse.Namespace) -> None: ctx.merge_contexts_from(harvest_context) if ctx._errors: - self.log.error('!!! warning "Errors during merge"') + self.log.error('Errors during merge') + self.errors.extend(ctx._errors) for ep, error in ctx._errors: self.log.info(" - %s: %s", ep.name, error) diff --git a/src/hermes/logger.py b/src/hermes/logger.py index cd8b28bc..7b6dd981 100644 --- a/src/hermes/logger.py +++ b/src/hermes/logger.py @@ -31,7 +31,7 @@ 'class': "logging.FileHandler", 'formatter': "logfile", 'level': "DEBUG", - 'filename': "./.hermes/hermes.log", + 'filename': "./hermes.log", }, 'auditfile': { @@ -50,12 +50,6 @@ }, } -# This dict caches all the different configuration sections already loaded -_config = { - # We need some basic logging configuration to get logging up and running at all - 'logging': _logging_config, -} - _loggers = {} @@ -64,14 +58,14 @@ def init_logging(): return # Make sure the directories to hold the log files exists (or else create) - pathlib.Path(_config['logging']['handlers']['logfile']['filename']).parent.mkdir(exist_ok=True, parents=True) - pathlib.Path(_config['logging']['handlers']['auditfile']['filename']).parent.mkdir(exist_ok=True, parents=True) + pathlib.Path(_logging_config['handlers']['logfile']['filename']).parent.mkdir(exist_ok=True, parents=True) + pathlib.Path(_logging_config['handlers']['auditfile']['filename']).parent.mkdir(exist_ok=True, parents=True) # Inintialize logging system import logging.config - logging.config.dictConfig(_config['logging']) - for log_name in _config['logging']['loggers']: + logging.config.dictConfig(_logging_config) + for log_name in _logging_config['loggers']: _loggers[log_name] = logging.getLogger(log_name) diff --git a/src/hermes/settings.py b/src/hermes/settings.py deleted file mode 100644 index fbae7b92..00000000 --- a/src/hermes/settings.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: 2024 German Aerospace Center (DLR) -# -# SPDX-License-Identifier: Apache-2.0 - -# SPDX-FileContributor: Sophie Kernchen -# SPDX-FileContributor: Michael Meinel - -import pathlib -from typing import Any, Dict, Tuple - -import toml -from pydantic.fields import FieldInfo -from pydantic_settings import PydanticBaseSettingsSource - - -class TomlConfigSettingsSource(PydanticBaseSettingsSource): - """ - A simple settings source class that loads variables from a TOML file - at the project's root. - - Here we happen to choose to use the `env_file_encoding` from Config - when reading `config.json` - """ - def __init__(self, settings_cls, config_path): - super().__init__(settings_cls) - self.__config_path = config_path - - def get_field_value(self, field: FieldInfo, field_name: str) -> Tuple[Any, str, bool]: - file_content_toml = toml.load(pathlib.Path(self.__config_path)) - field_value = file_content_toml.get(field_name) - return field_value, field_name, False - - def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any: - return value - - def __call__(self) -> Dict[str, Any]: - d: Dict[str, Any] = {} - for field_name, field in self.settings_cls.model_fields.items(): - field_value, field_key, value_is_complex = self.get_field_value(field, field_name) - field_value = self.prepare_field_value(field_name, field, field_value, value_is_complex) - if field_value is not None: - d[field_key] = field_value - - return d diff --git a/test/hermes_test/conftest.py b/test/hermes_test/conftest.py index 16f3b7a0..2d3e52b2 100644 --- a/test/hermes_test/conftest.py +++ b/test/hermes_test/conftest.py @@ -23,8 +23,10 @@ def __setitem__(self, path, data): def __enter__(self): self.test_path.mkdir(parents=True, exist_ok=True) + for file_name, data in self.test_files.items(): file_path = self.test_path / file_name + file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(data) os.chdir(self.test_path) diff --git a/test/hermes_test/test_cli.py b/test/hermes_test/test_cli.py index 9261c391..85b40e5e 100644 --- a/test/hermes_test/test_cli.py +++ b/test/hermes_test/test_cli.py @@ -9,23 +9,27 @@ from hermes.commands import cli -def test_hermes_full(capsys): +def test_hermes_full(): with pytest.raises(SystemExit) as se: cli.main() assert "choose from" in se -@pytest.mark.skip(reason="Needs update") def test_hermes_harvest(hermes_env): + hermes_env['hermes.toml'] = "" + with hermes_env: result = hermes_env.run("harvest") assert result.returncode == 0 -@pytest.mark.skip(reason="Needs update") def test_hermes_process(hermes_env): + hermes_env['hermes.toml'] = "" + hermes_env['.hermes/harvest/test.json'] = "" + with hermes_env: result = hermes_env.run("process") + print(result.stdout.read()) assert result.returncode == 0