Merge pull request #5571 from hjoliver/jinja2-scheduler-context

Add CYLC_ variables to template engine globals.

changes.d/5571.feat.md

@@ -0,0 +1 @@
+Make workflow `CYLC_` variables available to the template processor during parsing.

cylc/flow/parsec/empysupport.py

@@ -24,6 +24,7 @@
 import typing as t
 
 from cylc.flow.parsec.exceptions import EmPyError
+from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
 
 def empyprocess(
@@ -52,6 +53,12 @@ def empyprocess(
     ftempl = StringIO('\n'.join(flines))
     xtempl = StringIO()
     interpreter = em.Interpreter(output=em.UncloseableFile(xtempl))
+
+    # Add `CYLC_` environment variables to the global namespace.
+    interpreter.updateGlobals(
+        get_cylc_env_vars()
+    )
+
     try:
         interpreter.file(ftempl, '<template>', template_vars)
     except Exception as exc:

cylc/flow/parsec/fileparse.py

@@ -105,6 +105,27 @@
 )
 
 
+def get_cylc_env_vars() -> t.Dict[str, str]:
+    """Return a restricted dict of CYLC_ environment variables for templating.
+
+    The following variables are ignored because the do not necessarily reflect
+    the running code version (I might not use the "cylc" wrapper, or it might
+    select a different version):
+
+    CYLC_VERSION
+        Set as a template variable elsewhere, from the hardwired code version.
+
+    CYLC_ENV_NAME
+        Providing it as a template variable would just be misleading.
+    """
+    return {
+        key: val
+        for key, val in os.environ.items()
+        if key.startswith('CYLC_')
+        if key not in ["CYLC_VERSION", "CYLC_ENV_NAME"]
+    }
+
+
 def _concatenate(lines):
     """concatenate continuation lines"""
     index = 0
@@ -446,10 +467,9 @@ def read_and_proc(
         flines = inline(
             flines, fdir, fpath, viewcfg=viewcfg)
 
+    # Add the hardwired code version to template vars as CYLC_VERSION
     template_vars['CYLC_VERSION'] = __version__
-
     template_vars = merge_template_vars(template_vars, extra_vars)
-
     template_vars['CYLC_TEMPLATE_VARS'] = template_vars
 
     # Fail if templating_detected ≠ hashbang

cylc/flow/parsec/jinja2support.py

@@ -39,6 +39,7 @@
 
 from cylc.flow import LOG
 from cylc.flow.parsec.exceptions import Jinja2Error
+from cylc.flow.parsec.fileparse import get_cylc_env_vars
 
 TRACEBACK_LINENO = re.compile(
     r'\s+?File "(?P<file>.*)", line (?P<line>\d+), in .*template'
@@ -186,11 +187,15 @@ def jinja2environment(dir_=None):
                     envnsp[fname] = getattr(module, fname)
 
     # Import WORKFLOW HOST USER ENVIRONMENT into template:
-    # (usage e.g.: {{environ['HOME']}}).
+    # (Usage e.g.: {{environ['HOME']}}).
     env.globals['environ'] = os.environ
     env.globals['raise'] = raise_helper
     env.globals['assert'] = assert_helper
 
+    # Add `CYLC_` environment variables to the global namespace.
+    env.globals.update(
+        get_cylc_env_vars()
+    )
     return env
 
 
@@ -269,7 +274,6 @@ def jinja2process(
     # CALLERS SHOULD HANDLE JINJA2 TEMPLATESYNTAXERROR AND TEMPLATEERROR
     # AND TYPEERROR (e.g. for not using "|int" filter on number inputs.
     # Convert unicode to plain str, ToDo - still needed for parsec?)
-
     try:
         env = jinja2environment(dir_)
         template = env.from_string('\n'.join(flines[1:]))
@@ -300,16 +304,5 @@ def jinja2process(
             lines=get_error_lines(fpath, flines),
         )
 
-    flow_config = []
-    for line in lines:
-        # Jinja2 leaves blank lines where source lines contain
-        # only Jinja2 code; this matters if line continuation
-        # markers are involved, so we remove blank lines here.
-        if not line.strip():
-            continue
-            # restoring newlines here is only necessary for display by
-        # the cylc view command:
-        # ##flow_config.append(line + '\n')
-        flow_config.append(line)
-
-    return flow_config
+    # Ignore blank lines (lone Jinja2 statements leave blank lines behind)
+    return [line for line in lines if line.strip()]

cylc/flow/scripts/view.py

@@ -20,9 +20,17 @@
 
 Print a processed workflow configuration.
 
-Note:
-  This is different to `cylc config` which displays the parsed
-  configuration (as Cylc would see it).
+Print workflow configurations as processed before full parsing by Cylc. This
+includes Jinja2 or Empy template processing, and inlining of include-files.
+Some explanatory markup may also be requested.
+
+Warning:
+  This command will fail if `CYLC_` template variables are referenced
+  without default values, because they are only defined for full parsing.
+  E.g. (Jinja2): `{{CYLC_WORKFLOW_ID | default("not defined")}}`.
+
+See also `cylc config`, which displays the fully parsed configuration.
+
 """
 
 import asyncio
@@ -115,20 +123,21 @@ async def _main(options: 'Values', workflow_id: str) -> None:
         constraint='workflows',
     )
     # read in the flow.cylc file
-    viewcfg = {
-        'mark': options.mark,
-        'single': options.single,
-        'label': options.label,
-        'empy': options.empy or options.process,
-        'jinja2': options.jinja2 or options.process,
-        'contin': options.cat or options.process,
-        'inline': (options.inline or options.jinja2 or options.empy
-                   or options.process),
-    }
     for line in read_and_proc(
         flow_file,
         get_template_vars(options),
-        viewcfg=viewcfg,
+        viewcfg={
+            'mark': options.mark,
+            'single': options.single,
+            'label': options.label,
+            'empy': options.empy or options.process,
+            'jinja2': options.jinja2 or options.process,
+            'contin': options.cat or options.process,
+            'inline': (
+                options.jinja2 or options.empy or
+                options.inline or options.process
+            ),
+        },
         opts=options,
     ):
         print(line)

tests/functional/cylc-cat-log/01-remote.t

@@ -30,7 +30,9 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
 run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-run"
-workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}"
+workflow_run_ok "${TEST_NAME}" \
+    cylc play --debug --no-detach \
+        -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}"
 #-------------------------------------------------------------------------------
 TEST_NAME=${TEST_NAME_BASE}-task-out
 cylc cat-log -f o "${WORKFLOW_NAME}//1/a-task" >"${TEST_NAME}.out"
@@ -56,7 +58,7 @@ grep_ok "jumped over the lazy dog" "${TEST_NAME}.out"
 # remote
 TEST_NAME=${TEST_NAME_BASE}-task-status
 cylc cat-log -f s "${WORKFLOW_NAME}//1/a-task" >"${TEST_NAME}.out"
-grep_ok "CYLC_JOB_RUNNER_NAME=background" "${TEST_NAME}.out"
+grep_ok "CYLC_JOB_RUNNER_NAME=at" "${TEST_NAME}.out"
 #-------------------------------------------------------------------------------
 # local
 TEST_NAME=${TEST_NAME_BASE}-task-activity

tests/unit/parsec/test_fileparse.py

@@ -34,6 +34,7 @@
 from cylc.flow.parsec.fileparse import (
     _prepend_old_templatevars,
     _get_fpath_for_source,
+    get_cylc_env_vars,
     addict,
     addsect,
     multiline,
@@ -736,3 +737,23 @@ def test_get_fpath_for_source(tmp_path):
     opts.against_source = True
     assert _get_fpath_for_source(
         rundir / 'flow.cylc', opts) == str(srcdir / 'flow.cylc')
+
+
+def test_get_cylc_env_vars(monkeypatch):
+    """It should return CYLC env vars but not CYLC_VERSION or CYLC_ENV_NAME."""
+    monkeypatch.setattr(
+        'os.environ',
+        {
+            "CYLC_VERSION": "betwixt",
+            "CYLC_ENV_NAME": "between",
+            "CYLC_QUESTION": "que?", 
+            "CYLC_ANSWER": "42", 
+            "FOO": "foo"
+        }
+    )
+    assert (
+        get_cylc_env_vars() == {
+            "CYLC_QUESTION": "que?", 
+            "CYLC_ANSWER": "42", 
+        }
+    )

tests/unit/test_config.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import sys
 from optparse import Values
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type
 from pathlib import Path
@@ -34,6 +35,7 @@
     WorkflowConfigError,
     XtriggerConfigError,
 )
+from cylc.flow.parsec.exceptions import Jinja2Error, EmPyError
 from cylc.flow.scheduler_cli import RunOptions
 from cylc.flow.scripts.validate import ValidateOptions
 from cylc.flow.workflow_files import WorkflowFiles
@@ -1019,6 +1021,77 @@ def test_rsync_includes_will_not_accept_sub_directories(tmp_flow_config):
     assert "Directories can only be from the top level" in str(exc.value)
 
 
+@pytest.mark.parametrize(
+    'cylc_var, expected_err',
+    [
+        ["CYLC_WORKFLOW_NAME", None],
+        ["CYLC_BEEF_WELLINGTON", (Jinja2Error, "is undefined")],
+    ]
+)
+def test_jinja2_cylc_vars(tmp_flow_config, cylc_var, expected_err):
+    """Defined CYLC_ variables should be available to Jinja2 during parsing.
+
+    This test is not located in the jinja2_support unit test module because
+    CYLC_ variables are only defined during workflow config parsing.
+    """
+    reg = 'nodule'
+    flow_file = tmp_flow_config(reg, """#!Jinja2
+    # {{""" + cylc_var + """}}
+    [scheduler]
+        allow implicit tasks = True
+    [scheduling]
+        [[graph]]
+            R1 = foo
+    """)
+    if expected_err is None:
+        WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+    else:
+        with pytest.raises(expected_err[0]) as exc:
+            WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+        assert expected_err[1] in str(exc)
+
+
+@pytest.mark.parametrize(
+    'cylc_var, expected_err',
+    [
+        ["CYLC_WORKFLOW_NAME", None],
+        ["CYLC_BEEF_WELLINGTON", (EmPyError, "is not defined")],
+    ]
+)
+def test_empy_cylc_vars(tmp_flow_config, cylc_var, expected_err):
+    """Defined CYLC_ variables should be available to empy during parsing.
+
+    This test is not located in the empy_support unit test module because
+    CYLC_ variables are only defined during workflow config parsing.
+    """
+    reg = 'nodule'
+    flow_file = tmp_flow_config(reg, """#!empy
+    # @(""" + cylc_var + """)
+    [scheduler]
+        allow implicit tasks = True
+    [scheduling]
+        [[graph]]
+            R1 = foo
+    """)
+
+    # empy replaces sys.stdout with a "proxy". And pytest needs it for capture?
+    # (clue: "pytest --capture=no" avoids the error)
+    stdout = sys.stdout
+    sys.stdout._testProxy = lambda: ''
+    sys.stdout.pop = lambda _: ''
+    sys.stdout.push = lambda _: ''
+    sys.stdout.clear = lambda _: ''
+
+    if expected_err is None:
+        WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+    else:
+        with pytest.raises(expected_err[0]) as exc:
+            WorkflowConfig(workflow=reg, fpath=flow_file, options=Values())
+        assert expected_err[1] in str(exc)
+
+    sys.stdout = stdout
+
+
 def test_valid_rsync_includes_returns_correct_list(tmp_flow_config):
     """Test that the rsync includes in the correct """
     id_ = 'rsynctest'