feat(manager): pass custom env variables to workflow engines (#571)

reanahub · Feb 28, 2024 · cb9369b · cb9369b
1 parent 08ab9a3
commit cb9369b
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 7 deletions.
diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023 CERN.
+# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -16,6 +16,12 @@
 
 from reana_workflow_controller.version import __version__
 
+
+def _env_vars_dict_to_k8s_list(env_vars):
+    """Convert env vars stored as a dictionary into a k8s-compatible list."""
+    return [{"name": name, "value": str(value)} for name, value in env_vars.items()]
+
+
 SQLALCHEMY_TRACK_MODIFICATIONS = False
 """Track modifications flag."""
 
@@ -117,6 +123,27 @@
 ]
 """Common to all workflow engines environment variables."""
 
+
+WORKFLOW_ENGINE_CWL_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_CWL_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the CWL workflow engine container."""
+
+WORKFLOW_ENGINE_SERIAL_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SERIAL_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the serial workflow engine container."""
+
+WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the Snakemake workflow engine container."""
+
+WORKFLOW_ENGINE_YADAGE_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_YADAGE_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the Yadage workflow engine container."""
+
 DEBUG_ENV_VARS = (
     {
         "name": "WDB_SOCKET_SERVER",

diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py
@@ -1,11 +1,12 @@
 # This file is part of REANA.
-# Copyright (C) 2019, 2020, 2021, 2022, 2023 CERN.
+# Copyright (C) 2019, 2020, 2021, 2022, 2023, 2024 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
 """Workflow run manager interface."""
 import base64
+import copy
 import json
 import logging
 import os
@@ -71,6 +72,10 @@
     REANA_WORKFLOW_ENGINE_IMAGE_YADAGE,
     WORKFLOW_ENGINE_COMMON_ENV_VARS,
     DEBUG_ENV_VARS,
+    WORKFLOW_ENGINE_CWL_ENV_VARS,
+    WORKFLOW_ENGINE_SERIAL_ENV_VARS,
+    WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
+    WORKFLOW_ENGINE_YADAGE_ENV_VARS,
 )
 
 
@@ -92,7 +97,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_CWL_ENV_VARS,
         },
         "yadage": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_YADAGE),
@@ -105,7 +111,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_YADAGE_ENV_VARS,
         },
         "serial": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SERIAL),
@@ -117,7 +124,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_SERIAL_ENV_VARS,
         },
         "snakemake": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SNAKEMAKE),
@@ -129,7 +137,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
         },
     }
     """Mapping between engines and their basis configuration."""
@@ -220,7 +229,7 @@ def retrieve_required_cvmfs_repos(self):
 
     def _workflow_engine_env_vars(self):
         """Return necessary environment variables for the workflow engine."""
-        env_vars = list(
+        env_vars = copy.deepcopy(
             WorkflowRunManager.engine_mapping[self.workflow.type_][
                 "environment_variables"
             ]